tape: backup media catalogs

src/api2/tape/backup.rs

@@ -454,6 +454,20 @@ fn backup_worker(
 
     pool_writer.commit()?;
 
+    task_log!(worker, "append media catalog");
+
+    let uuid = pool_writer.load_writable_media(worker)?;
+    let done = pool_writer.append_catalog_archive(worker)?;
+    if !done {
+        task_log!(worker, "catalog does not fit on tape, writing to next volume");
+        pool_writer.set_media_status_full(&uuid)?;
+        pool_writer.load_writable_media(worker)?;
+        let done = pool_writer.append_catalog_archive(worker)?;
+        if !done {
+            bail!("write_catalog_archive failed on second media");
+        }
+    }
+
     if setup.export_media_set.unwrap_or(false) {
         pool_writer.export_media_set(worker)?;
     } else if setup.eject_media.unwrap_or(false) {

src/tape/file_formats/catalog_archive.rs

@@ -0,0 +1,89 @@
+use std::fs::File;
+use std::io::Read;
+
+use proxmox::{
+    sys::error::SysError,
+    tools::Uuid,
+};
+
+use crate::{
+    tape::{
+        TapeWrite,
+        file_formats::{
+            PROXMOX_TAPE_BLOCK_SIZE,
+            PROXMOX_BACKUP_CATALOG_ARCHIVE_MAGIC_1_0,
+            MediaContentHeader,
+            CatalogArchiveHeader,
+        },
+    },
+};
+
+/// Write a media catalog to the tape
+///
+/// Returns `Ok(Some(content_uuid))` on success, and `Ok(None)` if
+/// `LEOM` was detected before all data was written. The stream is
+/// marked inclomplete in that case and does not contain all data (The
+/// backup task must rewrite the whole file on the next media).
+///
+pub fn tape_write_catalog<'a>(
+    writer: &mut (dyn TapeWrite + 'a),
+    uuid: &Uuid,
+    media_set_uuid: &Uuid,
+    seq_nr: usize,
+    file: &mut File,
+) -> Result<Option<Uuid>, std::io::Error> {
+
+    let archive_header = CatalogArchiveHeader {
+        uuid: uuid.clone(),
+        media_set_uuid: media_set_uuid.clone(),
+        seq_nr: seq_nr as u64,
+    };
+
+    let header_data = serde_json::to_string_pretty(&archive_header)?.as_bytes().to_vec();
+
+    let header = MediaContentHeader::new(
+        PROXMOX_BACKUP_CATALOG_ARCHIVE_MAGIC_1_0, header_data.len() as u32);
+    let content_uuid: Uuid = header.uuid.into();
+
+    let leom = writer.write_header(&header, &header_data)?;
+    if leom {
+        writer.finish(true)?; // mark as incomplete
+        return Ok(None);
+    }
+
+    let mut file_copy_buffer = proxmox::tools::vec::undefined(PROXMOX_TAPE_BLOCK_SIZE);
+
+    let result: Result<(), std::io::Error> = proxmox::try_block!({
+
+        let file_size = file.metadata()?.len();
+        let mut remaining = file_size;
+
+        while remaining != 0 {
+            let got = file.read(&mut file_copy_buffer[..])?;
+            if got as u64 > remaining {
+                proxmox::io_bail!("catalog '{}' changed while reading", uuid);
+            }
+            writer.write_all(&file_copy_buffer[..got])?;
+            remaining -= got as u64;
+        }
+        if remaining > 0 {
+            proxmox::io_bail!("catalog '{}' shrunk while reading", uuid);
+        }
+        Ok(())
+    });
+    
+    match result {
+        Ok(()) => {
+            writer.finish(false)?;
+            Ok(Some(content_uuid))
+        }
+        Err(err) => {
+            if err.is_errno(nix::errno::Errno::ENOSPC) && writer.logical_end_of_media() {
+                writer.finish(true)?; // mark as incomplete
+                Ok(None)
+            } else {
+                Err(err)
+            }
+        }
+    }
+}

src/tape/file_formats/mod.rs

@@ -13,6 +13,9 @@ pub use chunk_archive::*;
 mod snapshot_archive;
 pub use snapshot_archive::*;
 
+mod catalog_archive;
+pub use catalog_archive::*;
+
 mod multi_volume_writer;
 pub use multi_volume_writer::*;
 

src/tape/media_pool.rs

@@ -101,9 +101,9 @@ impl MediaPool {
         self.force_media_availability = true;
     }
 
-    /// Returns the Uuid of the current media set
-    pub fn current_media_set(&self) -> &Uuid {
-        self.current_media_set.uuid()
+    /// Returns the the current media set
+    pub fn current_media_set(&self) -> &MediaSet {
+        &self.current_media_set
     }
 
     /// Creates a new instance using the media pool configuration

src/tape/pool_writer.rs

@@ -1,5 +1,6 @@
 use std::collections::HashSet;
 use std::path::Path;
+use std::fs::File;
 use std::time::SystemTime;
 use std::sync::{Arc, Mutex};
 
@@ -28,6 +29,7 @@ use crate::{
             MediaSetLabel,
             ChunkArchiveWriter,
             tape_write_snapshot_archive,
+            tape_write_catalog,
         },
         drive::{
             TapeDriver,
@@ -234,7 +236,12 @@ pub struct PoolWriter {
 
 impl PoolWriter {
 
-    pub fn new(mut pool: MediaPool, drive_name: &str, worker: &WorkerTask, notify_email: Option<String>) -> Result<Self, Error> {
+    pub fn new(
+        mut pool: MediaPool,
+        drive_name: &str,
+        worker: &WorkerTask,
+        notify_email: Option<String>,
+    ) -> Result<Self, Error> {
 
         let current_time = proxmox::tools::time::epoch_i64();
 
@@ -247,7 +254,8 @@ impl PoolWriter {
             );
         }
 
-        task_log!(worker, "media set uuid: {}", pool.current_media_set());
+        let media_set_uuid = pool.current_media_set().uuid();
+        task_log!(worker, "media set uuid: {}", media_set_uuid);
 
         let mut media_set_catalog = MediaSetCatalog::new();
 
@@ -404,7 +412,7 @@ impl PoolWriter {
             }
         }
 
-        let catalog = update_media_set_label(
+        let (catalog, new_media) = update_media_set_label(
             worker,
             drive.as_mut(),
             old_media_id.media_set_label,
@@ -424,9 +432,151 @@ impl PoolWriter {
 
         self.status = Some(PoolWriterState { drive, at_eom: false, bytes_written: 0 });
 
+        if new_media {
+            // add catalogs from previous media
+            self.append_media_set_catalogs(worker)?;
+        }
+
         Ok(media_uuid)
     }
 
+    fn open_catalog_file(uuid: &Uuid) -> Result<File, Error> {
+
+        let status_path = Path::new(TAPE_STATUS_DIR);
+        let mut path = status_path.to_owned();
+        path.push(uuid.to_string());
+        path.set_extension("log");
+
+        let file = std::fs::OpenOptions::new()
+            .read(true)
+            .open(&path)?;
+
+        Ok(file)
+    }
+
+    /// Move to EOM (if not already there), then write the current
+    /// catalog to the tape. On success, this return 'Ok(true)'.
+
+    /// Please note that this may fail when there is not enough space
+    /// on the media (return value 'Ok(false, _)'). In that case, the
+    /// archive is marked incomplete. The caller should mark the media
+    /// as full and try again using another media.
+    pub fn append_catalog_archive(
+        &mut self,
+        worker: &WorkerTask,
+    ) -> Result<bool, Error> {
+
+        let status = match self.status {
+            Some(ref mut status) => status,
+            None => bail!("PoolWriter - no media loaded"),
+        };
+
+        if !status.at_eom {
+            worker.log(String::from("moving to end of media"));
+            status.drive.move_to_eom()?;
+            status.at_eom = true;
+        }
+
+        let current_file_number = status.drive.current_file_number()?;
+        if current_file_number < 2 {
+            bail!("got strange file position number from drive ({})", current_file_number);
+        }
+
+        let catalog_builder = self.catalog_builder.lock().unwrap();
+
+        let catalog = match catalog_builder.catalog {
+            None => bail!("append_catalog_archive failed: no catalog - internal error"),
+            Some(ref catalog) => catalog,
+        };
+
+        let media_set = self.pool.current_media_set();
+
+        let media_list = media_set.media_list();
+        let uuid = match media_list.last() {
+            None => bail!("got empty media list - internal error"),
+            Some(None) => bail!("got incomplete media list - internal error"),
+            Some(Some(last_uuid)) => {
+                if last_uuid != catalog.uuid() {
+                    bail!("got wrong media - internal error");
+                }
+                last_uuid
+            }
+        };
+
+        let seq_nr = media_list.len() - 1;
+
+        let mut writer: Box<dyn TapeWrite> = status.drive.write_file()?;
+
+        let mut file = Self::open_catalog_file(uuid)?;
+
+        let done = tape_write_catalog(
+            writer.as_mut(),
+            uuid,
+            media_set.uuid(),
+            seq_nr,
+            &mut file,
+        )?.is_some();
+
+        Ok(done)
+    }
+
+    // Append catalogs for all previous media in set (without last)
+    fn append_media_set_catalogs(
+        &mut self,
+        worker: &WorkerTask,
+    ) -> Result<(), Error> {
+
+        let media_set = self.pool.current_media_set();
+
+        let mut media_list = &media_set.media_list()[..];
+        if media_list.len() < 2 {
+            return Ok(());
+        }
+        media_list = &media_list[..(media_list.len()-1)];
+
+        let status = match self.status {
+            Some(ref mut status) => status,
+            None => bail!("PoolWriter - no media loaded"),
+        };
+
+        if !status.at_eom {
+            worker.log(String::from("moving to end of media"));
+            status.drive.move_to_eom()?;
+            status.at_eom = true;
+        }
+
+        let current_file_number = status.drive.current_file_number()?;
+        if current_file_number < 2 {
+            bail!("got strange file position number from drive ({})", current_file_number);
+        }
+
+        for (seq_nr, uuid) in media_list.iter().enumerate() {
+
+            let uuid = match uuid {
+                None => bail!("got incomplete media list - internal error"),
+                Some(uuid) => uuid,
+            };
+
+            let mut writer: Box<dyn TapeWrite> = status.drive.write_file()?;
+
+            let mut file = Self::open_catalog_file(uuid)?;
+
+            task_log!(worker, "write catalog for previous media: {}", uuid);
+
+            if tape_write_catalog(
+                writer.as_mut(),
+                uuid,
+                media_set.uuid(),
+                seq_nr,
+                &mut file,
+            )?.is_none() {
+                bail!("got EOM while writing start catalog");
+            }
+        }
+
+        Ok(())
+    }
+
     /// Move to EOM (if not already there), then creates a new snapshot
     /// archive writing specified files (as .pxar) into it. On
     /// success, this return 'Ok(true)' and the media catalog gets
@@ -618,7 +768,7 @@ fn update_media_set_label(
     drive: &mut dyn TapeDriver,
     old_set: Option<MediaSetLabel>,
     media_id: &MediaId,
-) -> Result<MediaCatalog, Error> {
+) -> Result<(MediaCatalog, bool), Error> {
 
     let media_catalog;
 
@@ -641,11 +791,12 @@ fn update_media_set_label(
 
     let status_path = Path::new(TAPE_STATUS_DIR);
 
-    match old_set {
+    let new_media = match old_set {
         None => {
             worker.log("wrinting new media set label".to_string());
             drive.write_media_set_label(new_set, key_config.as_ref())?;
             media_catalog = MediaCatalog::overwrite(status_path, media_id, false)?;
+            true
         }
         Some(media_set_label) => {
             if new_set.uuid == media_set_label.uuid {
@@ -657,6 +808,10 @@ fn update_media_set_label(
                     bail!("detected changed encryption fingerprint - internal error");
                 }
                 media_catalog = MediaCatalog::open(status_path, &media_id, true, false)?;
+
+                // todo: verify last content/media_catalog somehow?
+
+                false
             } else {
                 worker.log(
                     format!("wrinting new media set label (overwrite '{}/{}')",
@@ -665,11 +820,10 @@ fn update_media_set_label(
 
                 drive.write_media_set_label(new_set, key_config.as_ref())?;
                 media_catalog = MediaCatalog::overwrite(status_path, media_id, false)?;
+                true
             }
         }
-    }
+    };
 
-    // todo: verify last content/media_catalog somehow?
-
-    Ok(media_catalog)
+    Ok((media_catalog, new_media))
 }