tape: add media state database

src/api2/tape/changer.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 use anyhow::Error;
 use serde_json::Value;
 
@@ -14,9 +16,14 @@ use crate::{
         MtxEntryKind,
     },
     tape::{
+        TAPE_STATUS_DIR,
         ElementStatus,
+        OnlineStatusMap,
+        Inventory,
+        MediaStateDatabase,
         linux_tape_changer_list,
         mtx_status,
+        mtx_status_to_online_set,
         mtx_transfer,
     },
 };
@@ -47,8 +54,7 @@ pub fn get_status(name: String) -> Result<Vec<MtxStatusEntry>, Error> {
 
     let status = mtx_status(&data.path)?;
 
-    /* todo: update persistent state
+    let state_path = Path::new(TAPE_STATUS_DIR);
-    let state_path = Path::new(MEDIA_POOL_STATUS_DIR);
     let inventory = Inventory::load(state_path)?;
 
     let mut map = OnlineStatusMap::new(&config)?;
@@ -57,7 +63,6 @@ pub fn get_status(name: String) -> Result<Vec<MtxStatusEntry>, Error> {
 
     let mut state_db = MediaStateDatabase::load(state_path)?;
     state_db.update_online_status(&map)?;
-     */
 
     let mut list = Vec::new();
 

src/api2/types/tape/media_status.rs

@@ -0,0 +1,21 @@
+use ::serde::{Deserialize, Serialize};
+
+use proxmox::api::api;
+
+#[api()]
+/// Media status
+#[derive(Debug, PartialEq, Copy, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// Media Status
+pub enum MediaStatus {
+    /// Media is ready to be written
+    Writable,
+    /// Media is full (contains data)
+    Full,
+    /// Media is marked as unknown, needs rescan
+    Unknown,
+    /// Media is marked as damaged
+    Damaged,
+    /// Media is marked as retired
+    Retired,
+}

src/api2/types/tape/mod.rs

@@ -8,3 +8,6 @@ pub use drive::*;
 
 mod media_pool;
 pub use media_pool::*;
+
+mod media_status;
+pub use media_status::*;

src/bin/proxmox-backup-api.rs

@@ -38,6 +38,7 @@ async fn run() -> Result<(), Error> {
 
     proxmox_backup::rrd::create_rrdb_dir()?;
     proxmox_backup::server::jobstate::create_jobstate_dir()?;
+    proxmox_backup::tape::create_tape_status_dir()?;
 
     if let Err(err) = generate_auth_key() {
         bail!("unable to generate auth key - {}", err);

src/tape/inventory.rs

@@ -28,7 +28,7 @@ use crate::{
         RetentionPolicy,
     },
     tape::{
-        MEDIA_POOL_STATUS_DIR,
+        TAPE_STATUS_DIR,
         file_formats::{
             DriveLabel,
             MediaSetLabel,
@@ -205,8 +205,16 @@ impl Inventory {
     fn replace_file(&self) -> Result<(), Error> {
         let list: Vec<&MediaId> = self.map.values().collect();
         let raw = serde_json::to_string_pretty(&serde_json::to_value(list)?)?;
-        let options = CreateOptions::new();
+
+        let backup_user = crate::backup::backup_user()?;
+        let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640);
+        let options = CreateOptions::new()
+            .perm(mode)
+            .owner(backup_user.uid)
+            .group(backup_user.gid);
+
         replace_file(&self.inventory_path, raw.as_bytes(), options)?;
+
         Ok(())
     }
 
@@ -605,7 +613,7 @@ pub fn complete_media_uuid(
     _param: &HashMap<String, String>,
 ) -> Vec<String> {
 
-    let inventory = match Inventory::load(Path::new(MEDIA_POOL_STATUS_DIR)) {
+    let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
         Ok(inventory) => inventory,
         Err(_) => return Vec::new(),
     };
@@ -619,7 +627,7 @@ pub fn complete_media_set_uuid(
     _param: &HashMap<String, String>,
 ) -> Vec<String> {
 
-    let inventory = match Inventory::load(Path::new(MEDIA_POOL_STATUS_DIR)) {
+    let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
         Ok(inventory) => inventory,
         Err(_) => return Vec::new(),
     };
@@ -635,7 +643,7 @@ pub fn complete_media_changer_id(
     _param: &HashMap<String, String>,
 ) -> Vec<String> {
 
-    let inventory = match Inventory::load(Path::new(MEDIA_POOL_STATUS_DIR)) {
+    let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
         Ok(inventory) => inventory,
         Err(_) => return Vec::new(),
     };

src/tape/media_state_database.rs

@@ -0,0 +1,224 @@
+use std::path::{Path, PathBuf};
+use std::collections::BTreeMap;
+
+use anyhow::Error;
+use ::serde::{Deserialize, Serialize};
+use serde_json::json;
+
+use proxmox::tools::{
+    Uuid,
+    fs::{
+        open_file_locked,
+        replace_file,
+        file_get_json,
+        CreateOptions,
+    },
+};
+
+use crate::{
+    tape::{
+        OnlineStatusMap,
+    },
+    api2::types::{
+        MediaStatus,
+    },
+};
+
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+/// Media location
+pub enum MediaLocation {
+    /// Ready for use (inside tape library)
+    Online(String),
+    /// Local available, but need to be mounted (insert into tape
+    /// drive)
+    Offline,
+    /// Media is inside a Vault
+    Vault(String),
+}
+
+#[derive(Serialize,Deserialize)]
+struct MediaStateEntry {
+    u: Uuid,
+    #[serde(skip_serializing_if="Option::is_none")]
+    l: Option<MediaLocation>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    s: Option<MediaStatus>,
+}
+
+impl MediaStateEntry {
+    fn new(uuid: Uuid) -> Self {
+        MediaStateEntry { u: uuid, l: None, s: None }
+    }
+}
+
+/// Stores MediaLocation and MediaState persistently
+pub struct MediaStateDatabase {
+
+    map: BTreeMap<Uuid, MediaStateEntry>,
+
+    database_path: PathBuf,
+    lockfile_path: PathBuf,
+}
+
+impl MediaStateDatabase {
+
+    pub const MEDIA_STATUS_DATABASE_FILENAME: &'static str = "media-status-db.json";
+    pub const MEDIA_STATUS_DATABASE_LOCKFILE: &'static str = ".media-status-db.lck";
+
+
+    /// Lock the database
+    pub fn lock(&self) -> Result<std::fs::File, Error> {
+        open_file_locked(&self.lockfile_path, std::time::Duration::new(10, 0), true)
+    }
+
+    /// Returns status and location with reasonable defaults.
+    ///
+    /// Default status is 'MediaStatus::Unknown'.
+    /// Default location is 'MediaLocation::Offline'.
+    pub fn status_and_location(&self, uuid: &Uuid) -> (MediaStatus, MediaLocation) {
+
+        match self.map.get(uuid) {
+            None => {
+                // no info stored - assume media is writable/offline
+                (MediaStatus::Unknown, MediaLocation::Offline)
+            }
+            Some(entry) => {
+                let location = entry.l.clone().unwrap_or(MediaLocation::Offline);
+                let status = entry.s.unwrap_or(MediaStatus::Unknown);
+                (status, location)
+            }
+        }
+    }
+
+    fn load_media_db(path: &Path) -> Result<BTreeMap<Uuid, MediaStateEntry>, Error> {
+
+        let data = file_get_json(path, Some(json!([])))?;
+        let list: Vec<MediaStateEntry> = serde_json::from_value(data)?;
+
+        let mut map = BTreeMap::new();
+        for entry in list.into_iter() {
+            map.insert(entry.u.clone(), entry);
+        }
+
+        Ok(map)
+    }
+
+    /// Load the database into memory
+    pub fn load(base_path: &Path) -> Result<MediaStateDatabase, Error> {
+
+        let mut database_path = base_path.to_owned();
+        database_path.push(Self::MEDIA_STATUS_DATABASE_FILENAME);
+
+        let mut lockfile_path = base_path.to_owned();
+        lockfile_path.push(Self::MEDIA_STATUS_DATABASE_LOCKFILE);
+
+        Ok(MediaStateDatabase {
+            map: Self::load_media_db(&database_path)?,
+            database_path,
+            lockfile_path,
+        })
+    }
+
+    /// Lock database, reload database, set status to Full, store database
+    pub fn set_media_status_full(&mut self, uuid: &Uuid) -> Result<(), Error> {
+        let _lock = self.lock()?;
+        self.map = Self::load_media_db(&self.database_path)?;
+        let entry = self.map.entry(uuid.clone()).or_insert(MediaStateEntry::new(uuid.clone()));
+        entry.s = Some(MediaStatus::Full);
+        self.store()
+    }
+
+    /// Update online status
+    pub fn update_online_status(&mut self, online_map: &OnlineStatusMap) -> Result<(), Error> {
+        let _lock = self.lock()?;
+        self.map = Self::load_media_db(&self.database_path)?;
+
+        for (_uuid, entry) in self.map.iter_mut() {
+            if let Some(changer_name) = online_map.lookup_changer(&entry.u) {
+                entry.l = Some(MediaLocation::Online(changer_name.to_string()));
+            } else {
+                if let Some(MediaLocation::Online(ref changer_name)) = entry.l {
+                    match online_map.online_map(changer_name) {
+                        None => {
+                            // no such changer device
+                            entry.l = Some(MediaLocation::Offline);
+                        }
+                        Some(None) => {
+                            // got no info - do nothing
+                        }
+                        Some(Some(_)) => {
+                            // media changer changed
+                            entry.l = Some(MediaLocation::Offline);
+                        }
+                    }
+                }
+            }
+        }
+
+        for (uuid, changer_name) in online_map.changer_map() {
+            if self.map.contains_key(uuid) { continue; }
+            let mut entry = MediaStateEntry::new(uuid.clone());
+            entry.l = Some(MediaLocation::Online(changer_name.to_string()));
+            self.map.insert(uuid.clone(), entry);
+        }
+
+        self.store()
+    }
+
+    /// Lock database, reload database, set status to Damaged, store database
+    pub fn set_media_status_damaged(&mut self, uuid: &Uuid) -> Result<(), Error> {
+        let _lock = self.lock()?;
+        self.map = Self::load_media_db(&self.database_path)?;
+        let entry = self.map.entry(uuid.clone()).or_insert(MediaStateEntry::new(uuid.clone()));
+        entry.s = Some(MediaStatus::Damaged);
+        self.store()
+    }
+
+    /// Lock database, reload database, set status to None, store database
+    pub fn clear_media_status(&mut self, uuid: &Uuid) -> Result<(), Error> {
+        let _lock = self.lock()?;
+        self.map = Self::load_media_db(&self.database_path)?;
+        let entry = self.map.entry(uuid.clone()).or_insert(MediaStateEntry::new(uuid.clone()));
+        entry.s = None ;
+        self.store()
+    }
+
+    /// Lock database, reload database, set location to vault, store database
+    pub fn set_media_location_vault(&mut self, uuid: &Uuid, vault: &str) -> Result<(), Error> {
+        let _lock = self.lock()?;
+        self.map = Self::load_media_db(&self.database_path)?;
+        let entry = self.map.entry(uuid.clone()).or_insert(MediaStateEntry::new(uuid.clone()));
+        entry.l = Some(MediaLocation::Vault(vault.to_string()));
+        self.store()
+    }
+
+    /// Lock database, reload database, set location to offline, store database
+    pub fn set_media_location_offline(&mut self, uuid: &Uuid) -> Result<(), Error> {
+        let _lock = self.lock()?;
+        self.map = Self::load_media_db(&self.database_path)?;
+        let entry = self.map.entry(uuid.clone()).or_insert(MediaStateEntry::new(uuid.clone()));
+        entry.l = Some(MediaLocation::Offline);
+        self.store()
+    }
+
+    fn store(&self) -> Result<(), Error> {
+
+        let mut list = Vec::new();
+        for entry in self.map.values() {
+            list.push(entry);
+        }
+
+        let raw = serde_json::to_string_pretty(&serde_json::to_value(list)?)?;
+
+        let backup_user = crate::backup::backup_user()?;
+        let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640);
+        let options = CreateOptions::new()
+            .perm(mode)
+            .owner(backup_user.uid)
+            .group(backup_user.gid);
+
+        replace_file(&self.database_path, raw.as_bytes(), options)?;
+
+        Ok(())
+    }
+}

src/tape/mod.rs

@@ -1,3 +1,10 @@
+use anyhow::{format_err, Error};
+
+use proxmox::tools::fs::{
+    create_path,
+    CreateOptions,
+};
+
 pub mod file_formats;
 
 mod tape_write;
@@ -18,8 +25,14 @@ pub use changer::*;
 mod drive;
 pub use drive::*;
 
-/// Directory path where we stora all status information
+mod media_state_database;
-pub const MEDIA_POOL_STATUS_DIR: &str = "/var/lib/proxmox-backup/mediapool";
+pub use media_state_database::*;
+
+mod online_status_map;
+pub use online_status_map::*;
+
+/// Directory path where we store all tape status information
+pub const TAPE_STATUS_DIR: &str = "/var/lib/proxmox-backup/tape";
 
 /// We limit chunk archive size, so that we can faster restore a
 /// specific chunk (The catalog only store file numbers, so we
@@ -28,3 +41,19 @@ pub const MAX_CHUNK_ARCHIVE_SIZE: usize = 4*1024*1024*1024; // 4GB for now
 
 /// To improve performance, we need to avoid tape drive buffer flush.
 pub const COMMIT_BLOCK_SIZE: usize = 128*1024*1024*1024; // 128 GiB
+
+
+/// Create tape status dir with correct permission
+pub fn create_tape_status_dir() -> Result<(), Error> {
+    let backup_user = crate::backup::backup_user()?;
+    let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640);
+    let opts = CreateOptions::new()
+        .perm(mode)
+        .owner(backup_user.uid)
+        .group(backup_user.gid);
+
+    create_path(TAPE_STATUS_DIR, None, Some(opts))
+        .map_err(|err: Error| format_err!("unable to create tape status dir - {}", err))?;
+
+    Ok(())
+}

src/tape/online_status_map.rs

@@ -0,0 +1,164 @@
+use std::path::Path;
+use std::collections::{HashMap, HashSet};
+
+use anyhow::{bail, Error};
+
+use proxmox::tools::Uuid;
+use proxmox::api::section_config::SectionConfigData;
+
+use crate::{
+    api2::types::{
+        VirtualTapeDrive,
+        ScsiTapeChanger,
+    },
+    tape::{
+        MediaChange,
+        Inventory,
+        MediaStateDatabase,
+        mtx_status,
+        mtx_status_to_online_set,
+    },
+};
+
+/// Helper to update media online status
+///
+/// A tape media is considered online if it is accessible by a changer
+/// device. This class can store the list of available changes,
+/// together with the accessible media ids.
+pub struct OnlineStatusMap {
+    map: HashMap<String, Option<HashSet<Uuid>>>,
+    changer_map: HashMap<Uuid, String>,
+}
+
+impl OnlineStatusMap {
+
+    /// Creates a new instance with one map entry for each configured
+    /// changer (or 'VirtualTapeDrive', which has an internal
+    /// changer). The map entry is set to 'None' to indicate that we
+    /// do not have information about the online status.
+    pub fn new(config: &SectionConfigData) -> Result<Self, Error> {
+
+        let mut map = HashMap::new();
+
+        let changers: Vec<ScsiTapeChanger> = config.convert_to_typed_array("changer")?;
+        for changer in changers {
+            map.insert(changer.name.clone(), None);
+        }
+
+        let vtapes: Vec<VirtualTapeDrive> = config.convert_to_typed_array("virtual")?;
+        for vtape in vtapes {
+            map.insert(vtape.name.clone(), None);
+        }
+
+        Ok(Self { map, changer_map: HashMap::new() })
+    }
+
+    /// Returns the assiciated changer name for a media.
+    pub fn lookup_changer(&self, uuid: &Uuid) -> Option<&String> {
+        self.changer_map.get(uuid)
+    }
+
+    /// Returns the map which assiciates media uuids with changer names.
+    pub fn changer_map(&self) -> &HashMap<Uuid, String> {
+        &self.changer_map
+    }
+
+    /// Returns the set of online media for the specified changer.
+    pub fn online_map(&self, changer_name: &str) -> Option<&Option<HashSet<Uuid>>> {
+        self.map.get(changer_name)
+    }
+
+    /// Update the online set for the specified changer
+    pub fn update_online_status(&mut self, changer_name: &str, online_set: HashSet<Uuid>) -> Result<(), Error> {
+
+        match self.map.get(changer_name) {
+            None => bail!("no such changer '{}' device", changer_name),
+            Some(None) => { /* Ok */ },
+            Some(Some(_)) => {
+                // do not allow updates to keep self.changer_map consistent
+                bail!("update_online_status '{}' called twice", changer_name);
+            }
+        }
+
+        for uuid in online_set.iter() {
+            self.changer_map.insert(uuid.clone(), changer_name.to_string());
+        }
+
+        self.map.insert(changer_name.to_string(), Some(online_set));
+
+        Ok(())
+    }
+}
+
+/// Update online media status
+///
+/// Simply ask all changer devices.
+pub fn update_online_status(state_path: &Path) -> Result<OnlineStatusMap, Error> {
+
+    let (config, _digest) = crate::config::drive::config()?;
+
+    let inventory = Inventory::load(state_path)?;
+
+    let changers: Vec<ScsiTapeChanger> = config.convert_to_typed_array("changer")?;
+
+    let mut map = OnlineStatusMap::new(&config)?;
+
+    for changer in changers {
+        let status = match mtx_status(&changer.path) {
+            Ok(status) => status,
+            Err(err) => {
+                eprintln!("unable to get changer '{}' status - {}", changer.name, err);
+                continue;
+            }
+        };
+
+        let online_set = mtx_status_to_online_set(&status, &inventory);
+        map.update_online_status(&changer.name, online_set)?;
+    }
+
+    let vtapes: Vec<VirtualTapeDrive> = config.convert_to_typed_array("virtual")?;
+    for vtape in vtapes {
+        let media_list = match vtape.list_media_changer_ids() {
+            Ok(media_list) => media_list,
+            Err(err) => {
+                eprintln!("unable to get changer '{}' status - {}", vtape.name, err);
+                continue;
+            }
+        };
+
+        let mut online_set = HashSet::new();
+        for changer_id in media_list {
+            if let Some(media_id) = inventory.find_media_by_changer_id(&changer_id) {
+                online_set.insert(media_id.label.uuid.clone());
+            }
+        }
+        map.update_online_status(&vtape.name, online_set)?;
+    }
+
+    let mut state_db = MediaStateDatabase::load(state_path)?;
+    state_db.update_online_status(&map)?;
+
+    Ok(map)
+}
+
+/// Update online media status with data from a single changer device
+pub fn update_changer_online_status(
+    drive_config: &SectionConfigData,
+    inventory: &mut Inventory,
+    state_db: &mut MediaStateDatabase,
+    changer_name: &str,
+    changer_id_list: &Vec<String>,
+) -> Result<(), Error> {
+
+    let mut online_map = OnlineStatusMap::new(drive_config)?;
+    let mut online_set = HashSet::new();
+    for changer_id in changer_id_list.iter() {
+        if let Some(media_id) = inventory.find_media_by_changer_id(&changer_id) {
+            online_set.insert(media_id.label.uuid.clone());
+        }
+    }
+    online_map.update_online_status(&changer_name, online_set)?;
+    state_db.update_online_status(&online_map)?;
+
+    Ok(())
+}