diff --git a/src/api2/tape/backup.rs b/src/api2/tape/backup.rs index 690b6855..e4fba59c 100644 --- a/src/api2/tape/backup.rs +++ b/src/api2/tape/backup.rs @@ -454,6 +454,20 @@ fn backup_worker( pool_writer.commit()?; + task_log!(worker, "append media catalog"); + + let uuid = pool_writer.load_writable_media(worker)?; + let done = pool_writer.append_catalog_archive(worker)?; + if !done { + task_log!(worker, "catalog does not fit on tape, writing to next volume"); + pool_writer.set_media_status_full(&uuid)?; + pool_writer.load_writable_media(worker)?; + let done = pool_writer.append_catalog_archive(worker)?; + if !done { + bail!("write_catalog_archive failed on second media"); + } + } + if setup.export_media_set.unwrap_or(false) { pool_writer.export_media_set(worker)?; } else if setup.eject_media.unwrap_or(false) { diff --git a/src/tape/file_formats/catalog_archive.rs b/src/tape/file_formats/catalog_archive.rs new file mode 100644 index 00000000..d48a2ca5 --- /dev/null +++ b/src/tape/file_formats/catalog_archive.rs @@ -0,0 +1,89 @@ +use std::fs::File; +use std::io::Read; + +use proxmox::{ + sys::error::SysError, + tools::Uuid, +}; + +use crate::{ + tape::{ + TapeWrite, + file_formats::{ + PROXMOX_TAPE_BLOCK_SIZE, + PROXMOX_BACKUP_CATALOG_ARCHIVE_MAGIC_1_0, + MediaContentHeader, + CatalogArchiveHeader, + }, + }, +}; + +/// Write a media catalog to the tape +/// +/// Returns `Ok(Some(content_uuid))` on success, and `Ok(None)` if +/// `LEOM` was detected before all data was written. The stream is +/// marked inclomplete in that case and does not contain all data (The +/// backup task must rewrite the whole file on the next media). +/// +pub fn tape_write_catalog<'a>( + writer: &mut (dyn TapeWrite + 'a), + uuid: &Uuid, + media_set_uuid: &Uuid, + seq_nr: usize, + file: &mut File, +) -> Result, std::io::Error> { + + let archive_header = CatalogArchiveHeader { + uuid: uuid.clone(), + media_set_uuid: media_set_uuid.clone(), + seq_nr: seq_nr as u64, + }; + + let header_data = serde_json::to_string_pretty(&archive_header)?.as_bytes().to_vec(); + + let header = MediaContentHeader::new( + PROXMOX_BACKUP_CATALOG_ARCHIVE_MAGIC_1_0, header_data.len() as u32); + let content_uuid: Uuid = header.uuid.into(); + + let leom = writer.write_header(&header, &header_data)?; + if leom { + writer.finish(true)?; // mark as incomplete + return Ok(None); + } + + let mut file_copy_buffer = proxmox::tools::vec::undefined(PROXMOX_TAPE_BLOCK_SIZE); + + let result: Result<(), std::io::Error> = proxmox::try_block!({ + + let file_size = file.metadata()?.len(); + let mut remaining = file_size; + + while remaining != 0 { + let got = file.read(&mut file_copy_buffer[..])?; + if got as u64 > remaining { + proxmox::io_bail!("catalog '{}' changed while reading", uuid); + } + writer.write_all(&file_copy_buffer[..got])?; + remaining -= got as u64; + } + if remaining > 0 { + proxmox::io_bail!("catalog '{}' shrunk while reading", uuid); + } + Ok(()) + }); + + match result { + Ok(()) => { + writer.finish(false)?; + Ok(Some(content_uuid)) + } + Err(err) => { + if err.is_errno(nix::errno::Errno::ENOSPC) && writer.logical_end_of_media() { + writer.finish(true)?; // mark as incomplete + Ok(None) + } else { + Err(err) + } + } + } +} diff --git a/src/tape/file_formats/mod.rs b/src/tape/file_formats/mod.rs index b60fbe7b..e94818b2 100644 --- a/src/tape/file_formats/mod.rs +++ b/src/tape/file_formats/mod.rs @@ -13,6 +13,9 @@ pub use chunk_archive::*; mod snapshot_archive; pub use snapshot_archive::*; +mod catalog_archive; +pub use catalog_archive::*; + mod multi_volume_writer; pub use multi_volume_writer::*; diff --git a/src/tape/media_pool.rs b/src/tape/media_pool.rs index a249da34..19debd2a 100644 --- a/src/tape/media_pool.rs +++ b/src/tape/media_pool.rs @@ -101,9 +101,9 @@ impl MediaPool { self.force_media_availability = true; } - /// Returns the Uuid of the current media set - pub fn current_media_set(&self) -> &Uuid { - self.current_media_set.uuid() + /// Returns the the current media set + pub fn current_media_set(&self) -> &MediaSet { + &self.current_media_set } /// Creates a new instance using the media pool configuration diff --git a/src/tape/pool_writer.rs b/src/tape/pool_writer.rs index 9c683861..12926945 100644 --- a/src/tape/pool_writer.rs +++ b/src/tape/pool_writer.rs @@ -1,5 +1,6 @@ use std::collections::HashSet; use std::path::Path; +use std::fs::File; use std::time::SystemTime; use std::sync::{Arc, Mutex}; @@ -28,6 +29,7 @@ use crate::{ MediaSetLabel, ChunkArchiveWriter, tape_write_snapshot_archive, + tape_write_catalog, }, drive::{ TapeDriver, @@ -234,7 +236,12 @@ pub struct PoolWriter { impl PoolWriter { - pub fn new(mut pool: MediaPool, drive_name: &str, worker: &WorkerTask, notify_email: Option) -> Result { + pub fn new( + mut pool: MediaPool, + drive_name: &str, + worker: &WorkerTask, + notify_email: Option, + ) -> Result { let current_time = proxmox::tools::time::epoch_i64(); @@ -247,7 +254,8 @@ impl PoolWriter { ); } - task_log!(worker, "media set uuid: {}", pool.current_media_set()); + let media_set_uuid = pool.current_media_set().uuid(); + task_log!(worker, "media set uuid: {}", media_set_uuid); let mut media_set_catalog = MediaSetCatalog::new(); @@ -404,7 +412,7 @@ impl PoolWriter { } } - let catalog = update_media_set_label( + let (catalog, new_media) = update_media_set_label( worker, drive.as_mut(), old_media_id.media_set_label, @@ -424,9 +432,151 @@ impl PoolWriter { self.status = Some(PoolWriterState { drive, at_eom: false, bytes_written: 0 }); + if new_media { + // add catalogs from previous media + self.append_media_set_catalogs(worker)?; + } + Ok(media_uuid) } + fn open_catalog_file(uuid: &Uuid) -> Result { + + let status_path = Path::new(TAPE_STATUS_DIR); + let mut path = status_path.to_owned(); + path.push(uuid.to_string()); + path.set_extension("log"); + + let file = std::fs::OpenOptions::new() + .read(true) + .open(&path)?; + + Ok(file) + } + + /// Move to EOM (if not already there), then write the current + /// catalog to the tape. On success, this return 'Ok(true)'. + + /// Please note that this may fail when there is not enough space + /// on the media (return value 'Ok(false, _)'). In that case, the + /// archive is marked incomplete. The caller should mark the media + /// as full and try again using another media. + pub fn append_catalog_archive( + &mut self, + worker: &WorkerTask, + ) -> Result { + + let status = match self.status { + Some(ref mut status) => status, + None => bail!("PoolWriter - no media loaded"), + }; + + if !status.at_eom { + worker.log(String::from("moving to end of media")); + status.drive.move_to_eom()?; + status.at_eom = true; + } + + let current_file_number = status.drive.current_file_number()?; + if current_file_number < 2 { + bail!("got strange file position number from drive ({})", current_file_number); + } + + let catalog_builder = self.catalog_builder.lock().unwrap(); + + let catalog = match catalog_builder.catalog { + None => bail!("append_catalog_archive failed: no catalog - internal error"), + Some(ref catalog) => catalog, + }; + + let media_set = self.pool.current_media_set(); + + let media_list = media_set.media_list(); + let uuid = match media_list.last() { + None => bail!("got empty media list - internal error"), + Some(None) => bail!("got incomplete media list - internal error"), + Some(Some(last_uuid)) => { + if last_uuid != catalog.uuid() { + bail!("got wrong media - internal error"); + } + last_uuid + } + }; + + let seq_nr = media_list.len() - 1; + + let mut writer: Box = status.drive.write_file()?; + + let mut file = Self::open_catalog_file(uuid)?; + + let done = tape_write_catalog( + writer.as_mut(), + uuid, + media_set.uuid(), + seq_nr, + &mut file, + )?.is_some(); + + Ok(done) + } + + // Append catalogs for all previous media in set (without last) + fn append_media_set_catalogs( + &mut self, + worker: &WorkerTask, + ) -> Result<(), Error> { + + let media_set = self.pool.current_media_set(); + + let mut media_list = &media_set.media_list()[..]; + if media_list.len() < 2 { + return Ok(()); + } + media_list = &media_list[..(media_list.len()-1)]; + + let status = match self.status { + Some(ref mut status) => status, + None => bail!("PoolWriter - no media loaded"), + }; + + if !status.at_eom { + worker.log(String::from("moving to end of media")); + status.drive.move_to_eom()?; + status.at_eom = true; + } + + let current_file_number = status.drive.current_file_number()?; + if current_file_number < 2 { + bail!("got strange file position number from drive ({})", current_file_number); + } + + for (seq_nr, uuid) in media_list.iter().enumerate() { + + let uuid = match uuid { + None => bail!("got incomplete media list - internal error"), + Some(uuid) => uuid, + }; + + let mut writer: Box = status.drive.write_file()?; + + let mut file = Self::open_catalog_file(uuid)?; + + task_log!(worker, "write catalog for previous media: {}", uuid); + + if tape_write_catalog( + writer.as_mut(), + uuid, + media_set.uuid(), + seq_nr, + &mut file, + )?.is_none() { + bail!("got EOM while writing start catalog"); + } + } + + Ok(()) + } + /// Move to EOM (if not already there), then creates a new snapshot /// archive writing specified files (as .pxar) into it. On /// success, this return 'Ok(true)' and the media catalog gets @@ -618,7 +768,7 @@ fn update_media_set_label( drive: &mut dyn TapeDriver, old_set: Option, media_id: &MediaId, -) -> Result { +) -> Result<(MediaCatalog, bool), Error> { let media_catalog; @@ -641,11 +791,12 @@ fn update_media_set_label( let status_path = Path::new(TAPE_STATUS_DIR); - match old_set { + let new_media = match old_set { None => { worker.log("wrinting new media set label".to_string()); drive.write_media_set_label(new_set, key_config.as_ref())?; media_catalog = MediaCatalog::overwrite(status_path, media_id, false)?; + true } Some(media_set_label) => { if new_set.uuid == media_set_label.uuid { @@ -657,6 +808,10 @@ fn update_media_set_label( bail!("detected changed encryption fingerprint - internal error"); } media_catalog = MediaCatalog::open(status_path, &media_id, true, false)?; + + // todo: verify last content/media_catalog somehow? + + false } else { worker.log( format!("wrinting new media set label (overwrite '{}/{}')", @@ -665,11 +820,10 @@ fn update_media_set_label( drive.write_media_set_label(new_set, key_config.as_ref())?; media_catalog = MediaCatalog::overwrite(status_path, media_id, false)?; + true } } - } + }; - // todo: verify last content/media_catalog somehow? - - Ok(media_catalog) + Ok((media_catalog, new_media)) }