tape: cleanup - move tape file readers/writers into src/tape/file_formats folder
This commit is contained in:
Dietmar Maurer
316
src/tape/file_formats/blocked_reader.rs
Normal file
316
src/tape/file_formats/blocked_reader.rs
Normal file
@ -0,0 +1,316 @@
|
||||
use std::io::Read;
|
||||
|
||||
use crate::tape::{
|
||||
TapeRead,
|
||||
tape_device_read_block,
|
||||
file_formats::{
|
||||
PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0,
|
||||
BlockHeader,
|
||||
BlockHeaderFlags,
|
||||
},
|
||||
};
|
||||
|
||||
/// Read a block stream generated by 'BlockWriter'.
|
||||
///
|
||||
/// This class implements 'TapeRead'. It always read whole blocks from
|
||||
/// the underlying reader, and does additional error checks:
|
||||
///
|
||||
/// - check magic number (detect streams not written by 'BlockWriter')
|
||||
/// - check block size
|
||||
/// - check block sequence numbers
|
||||
///
|
||||
/// The reader consumes the EOF mark after the data stream (if read to
|
||||
/// the end of the stream).
|
||||
pub struct BlockedReader<R> {
|
||||
reader: R,
|
||||
buffer: Box<BlockHeader>,
|
||||
seq_nr: u32,
|
||||
found_end_marker: bool,
|
||||
incomplete: bool,
|
||||
got_eod: bool,
|
||||
read_error: bool,
|
||||
read_pos: usize,
|
||||
}
|
||||
|
||||
impl <R: Read> BlockedReader<R> {
|
||||
|
||||
/// Create a new BlockedReader instance.
|
||||
///
|
||||
/// This tries to read the first block, and returns None if we are
|
||||
/// at EOT.
|
||||
pub fn open(mut reader: R) -> Result<Option<Self>, std::io::Error> {
|
||||
|
||||
let mut buffer = BlockHeader::new();
|
||||
|
||||
if !Self::read_block_frame(&mut buffer, &mut reader)? {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let (_size, found_end_marker) = Self::check_buffer(&buffer, 0)?;
|
||||
|
||||
let mut incomplete = false;
|
||||
let mut got_eod = false;
|
||||
|
||||
if found_end_marker {
|
||||
incomplete = buffer.flags.contains(BlockHeaderFlags::INCOMPLETE);
|
||||
Self::consume_eof_marker(&mut reader)?;
|
||||
got_eod = true;
|
||||
}
|
||||
|
||||
Ok(Some(Self {
|
||||
reader,
|
||||
buffer,
|
||||
found_end_marker,
|
||||
incomplete,
|
||||
got_eod,
|
||||
seq_nr: 1,
|
||||
read_error: false,
|
||||
read_pos: 0,
|
||||
}))
|
||||
}
|
||||
|
||||
fn check_buffer(buffer: &BlockHeader, seq_nr: u32) -> Result<(usize, bool), std::io::Error> {
|
||||
|
||||
if buffer.magic != PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0 {
|
||||
proxmox::io_bail!("detected tape block with wrong magic number - not written by proxmox tape");
|
||||
}
|
||||
|
||||
if seq_nr != buffer.seq_nr() {
|
||||
proxmox::io_bail!(
|
||||
"detected tape block with wrong seqence number ({} != {})",
|
||||
seq_nr, buffer.seq_nr())
|
||||
}
|
||||
|
||||
let size = buffer.size();
|
||||
let found_end_marker = buffer.flags.contains(BlockHeaderFlags::END_OF_STREAM);
|
||||
|
||||
if size > buffer.payload.len() {
|
||||
proxmox::io_bail!("detected tape block with wrong payload size ({} > {}", size, buffer.payload.len());
|
||||
} else if size == 0 && !found_end_marker {
|
||||
proxmox::io_bail!("detected tape block with zero payload size");
|
||||
}
|
||||
|
||||
|
||||
Ok((size, found_end_marker))
|
||||
}
|
||||
|
||||
fn read_block_frame(buffer: &mut BlockHeader, reader: &mut R) -> Result<bool, std::io::Error> {
|
||||
|
||||
let data = unsafe {
|
||||
std::slice::from_raw_parts_mut(
|
||||
(buffer as *mut BlockHeader) as *mut u8,
|
||||
BlockHeader::SIZE,
|
||||
)
|
||||
};
|
||||
|
||||
tape_device_read_block(reader, data)
|
||||
}
|
||||
|
||||
fn consume_eof_marker(reader: &mut R) -> Result<(), std::io::Error> {
|
||||
let mut tmp_buf = [0u8; 512]; // use a small buffer for testing EOF
|
||||
if tape_device_read_block(reader, &mut tmp_buf)? {
|
||||
proxmox::io_bail!("detected tape block after stream end marker");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_block(&mut self) -> Result<usize, std::io::Error> {
|
||||
|
||||
if !Self::read_block_frame(&mut self.buffer, &mut self.reader)? {
|
||||
self.got_eod = true;
|
||||
self.read_pos = self.buffer.payload.len();
|
||||
if !self.found_end_marker {
|
||||
proxmox::io_bail!("detected tape stream without end marker");
|
||||
}
|
||||
return Ok(0); // EOD
|
||||
}
|
||||
|
||||
let (size, found_end_marker) = Self::check_buffer(&self.buffer, self.seq_nr)?;
|
||||
self.seq_nr += 1;
|
||||
|
||||
if found_end_marker { // consume EOF mark
|
||||
self.found_end_marker = true;
|
||||
self.incomplete = self.buffer.flags.contains(BlockHeaderFlags::INCOMPLETE);
|
||||
Self::consume_eof_marker(&mut self.reader)?;
|
||||
self.got_eod = true;
|
||||
}
|
||||
|
||||
self.read_pos = 0;
|
||||
|
||||
Ok(size)
|
||||
}
|
||||
}
|
||||
|
||||
impl <R: Read> TapeRead for BlockedReader<R> {
|
||||
|
||||
fn is_incomplete(&self) -> Result<bool, std::io::Error> {
|
||||
if !self.got_eod {
|
||||
proxmox::io_bail!("is_incomplete failed: EOD not reached");
|
||||
}
|
||||
if !self.found_end_marker {
|
||||
proxmox::io_bail!("is_incomplete failed: no end marker found");
|
||||
}
|
||||
|
||||
Ok(self.incomplete)
|
||||
}
|
||||
|
||||
fn has_end_marker(&self) -> Result<bool, std::io::Error> {
|
||||
if !self.got_eod {
|
||||
proxmox::io_bail!("has_end_marker failed: EOD not reached");
|
||||
}
|
||||
|
||||
Ok(self.found_end_marker)
|
||||
}
|
||||
}
|
||||
|
||||
impl <R: Read> Read for BlockedReader<R> {
|
||||
|
||||
fn read(&mut self, buffer: &mut [u8]) -> Result<usize, std::io::Error> {
|
||||
|
||||
if self.read_error {
|
||||
proxmox::io_bail!("detected read after error - internal error");
|
||||
}
|
||||
|
||||
let mut buffer_size = self.buffer.size();
|
||||
let mut rest = (buffer_size as isize) - (self.read_pos as isize);
|
||||
|
||||
if rest <= 0 && !self.got_eod { // try to refill buffer
|
||||
buffer_size = match self.read_block() {
|
||||
Ok(len) => len,
|
||||
err => {
|
||||
self.read_error = true;
|
||||
return err;
|
||||
}
|
||||
};
|
||||
rest = buffer_size as isize;
|
||||
}
|
||||
|
||||
if rest <= 0 {
|
||||
Ok(0)
|
||||
} else {
|
||||
let copy_len = if (buffer.len() as isize) < rest {
|
||||
buffer.len()
|
||||
} else {
|
||||
rest as usize
|
||||
};
|
||||
buffer[..copy_len].copy_from_slice(
|
||||
&self.buffer.payload[self.read_pos..(self.read_pos + copy_len)]);
|
||||
self.read_pos += copy_len;
|
||||
Ok(copy_len)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use std::io::Read;
|
||||
use anyhow::Error;
|
||||
use crate::tape::{
|
||||
TapeWrite,
|
||||
file_formats::PROXMOX_TAPE_BLOCK_SIZE,
|
||||
helpers::{
|
||||
BlockedReader,
|
||||
BlockedWriter,
|
||||
},
|
||||
};
|
||||
|
||||
fn write_and_verify(data: &[u8]) -> Result<(), Error> {
|
||||
|
||||
let mut tape_data = Vec::new();
|
||||
|
||||
let mut writer = BlockedWriter::new(&mut tape_data);
|
||||
|
||||
writer.write_all(data)?;
|
||||
|
||||
writer.finish(false)?;
|
||||
|
||||
assert_eq!(
|
||||
tape_data.len(),
|
||||
((data.len() + PROXMOX_TAPE_BLOCK_SIZE)/PROXMOX_TAPE_BLOCK_SIZE)
|
||||
*PROXMOX_TAPE_BLOCK_SIZE
|
||||
);
|
||||
|
||||
let reader = &mut &tape_data[..];
|
||||
let mut reader = BlockedReader::open(reader)?.unwrap();
|
||||
|
||||
let mut read_data = Vec::with_capacity(PROXMOX_TAPE_BLOCK_SIZE);
|
||||
reader.read_to_end(&mut read_data)?;
|
||||
|
||||
assert_eq!(data.len(), read_data.len());
|
||||
|
||||
assert_eq!(data, &read_data[..]);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_stream() -> Result<(), Error> {
|
||||
write_and_verify(b"")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_data() -> Result<(), Error> {
|
||||
write_and_verify(b"ABC")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_data() -> Result<(), Error> {
|
||||
let data = proxmox::sys::linux::random_data(1024*1024*5)?;
|
||||
write_and_verify(&data)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_data() -> Result<(), Error> {
|
||||
let tape_data = Vec::new();
|
||||
let reader = &mut &tape_data[..];
|
||||
let reader = BlockedReader::open(reader)?;
|
||||
assert!(reader.is_none());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_end_marker() -> Result<(), Error> {
|
||||
let mut tape_data = Vec::new();
|
||||
{
|
||||
let mut writer = BlockedWriter::new(&mut tape_data);
|
||||
// write at least one block
|
||||
let data = proxmox::sys::linux::random_data(PROXMOX_TAPE_BLOCK_SIZE)?;
|
||||
writer.write_all(&data)?;
|
||||
// but do not call finish here
|
||||
}
|
||||
let reader = &mut &tape_data[..];
|
||||
let mut reader = BlockedReader::open(reader)?.unwrap();
|
||||
|
||||
let mut data = Vec::with_capacity(PROXMOX_TAPE_BLOCK_SIZE);
|
||||
assert!(reader.read_to_end(&mut data).is_err());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_read_buffer() -> Result<(), Error> {
|
||||
let mut tape_data = Vec::new();
|
||||
|
||||
let mut writer = BlockedWriter::new(&mut tape_data);
|
||||
|
||||
writer.write_all(b"ABC")?;
|
||||
|
||||
writer.finish(false)?;
|
||||
|
||||
let reader = &mut &tape_data[..];
|
||||
let mut reader = BlockedReader::open(reader)?.unwrap();
|
||||
|
||||
let mut buf = [0u8; 1];
|
||||
assert_eq!(reader.read(&mut buf)?, 1, "wrong byte count");
|
||||
assert_eq!(&buf, b"A");
|
||||
assert_eq!(reader.read(&mut buf)?, 1, "wrong byte count");
|
||||
assert_eq!(&buf, b"B");
|
||||
assert_eq!(reader.read(&mut buf)?, 1, "wrong byte count");
|
||||
assert_eq!(&buf, b"C");
|
||||
assert_eq!(reader.read(&mut buf)?, 0, "wrong byte count");
|
||||
assert_eq!(reader.read(&mut buf)?, 0, "wrong byte count");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
124
src/tape/file_formats/blocked_writer.rs
Normal file
124
src/tape/file_formats/blocked_writer.rs
Normal file
@ -0,0 +1,124 @@
|
||||
use std::io::Write;
|
||||
|
||||
use proxmox::tools::vec;
|
||||
|
||||
use crate::tape::{
|
||||
TapeWrite,
|
||||
tape_device_write_block,
|
||||
file_formats::{
|
||||
BlockHeader,
|
||||
BlockHeaderFlags,
|
||||
},
|
||||
};
|
||||
|
||||
/// Assemble and write blocks of data
|
||||
///
|
||||
/// This type implement 'TapeWrite'. Data written is assembled to
|
||||
/// equally sized blocks (see 'BlockHeader'), which are then written
|
||||
/// to the underlying writer.
|
||||
pub struct BlockedWriter<W> {
|
||||
writer: W,
|
||||
buffer: Box<BlockHeader>,
|
||||
buffer_pos: usize,
|
||||
seq_nr: u32,
|
||||
logical_end_of_media: bool,
|
||||
bytes_written: usize,
|
||||
}
|
||||
|
||||
impl <W: Write> BlockedWriter<W> {
|
||||
|
||||
/// Allow access to underlying writer
|
||||
pub fn writer_ref_mut(&mut self) -> &mut W {
|
||||
&mut self.writer
|
||||
}
|
||||
|
||||
/// Creates a new instance.
|
||||
pub fn new(writer: W) -> Self {
|
||||
Self {
|
||||
writer,
|
||||
buffer: BlockHeader::new(),
|
||||
buffer_pos: 0,
|
||||
seq_nr: 0,
|
||||
logical_end_of_media: false,
|
||||
bytes_written: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn write_block(buffer: &BlockHeader, writer: &mut W) -> Result<bool, std::io::Error> {
|
||||
|
||||
let data = unsafe {
|
||||
std::slice::from_raw_parts(
|
||||
(buffer as *const BlockHeader) as *const u8,
|
||||
BlockHeader::SIZE,
|
||||
)
|
||||
};
|
||||
tape_device_write_block(writer, data)
|
||||
}
|
||||
|
||||
fn write(&mut self, data: &[u8]) -> Result<usize, std::io::Error> {
|
||||
|
||||
if data.is_empty() { return Ok(0); }
|
||||
|
||||
let rest = self.buffer.payload.len() - self.buffer_pos;
|
||||
let bytes = if data.len() < rest { data.len() } else { rest };
|
||||
self.buffer.payload[self.buffer_pos..(self.buffer_pos+bytes)]
|
||||
.copy_from_slice(&data[..bytes]);
|
||||
|
||||
let rest = rest - bytes;
|
||||
|
||||
if rest == 0 {
|
||||
self.buffer.flags = BlockHeaderFlags::empty();
|
||||
self.buffer.set_size(self.buffer.payload.len());
|
||||
self.buffer.set_seq_nr(self.seq_nr);
|
||||
self.seq_nr += 1;
|
||||
let leom = Self::write_block(&self.buffer, &mut self.writer)?;
|
||||
if leom { self.logical_end_of_media = true; }
|
||||
self.buffer_pos = 0;
|
||||
self.bytes_written += BlockHeader::SIZE;
|
||||
|
||||
} else {
|
||||
self.buffer_pos += bytes;
|
||||
}
|
||||
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
impl <W: Write> TapeWrite for BlockedWriter<W> {
|
||||
|
||||
fn write_all(&mut self, mut data: &[u8]) -> Result<bool, std::io::Error> {
|
||||
while !data.is_empty() {
|
||||
match self.write(data) {
|
||||
Ok(n) => data = &data[n..],
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Ok(self.logical_end_of_media)
|
||||
}
|
||||
|
||||
fn bytes_written(&self) -> usize {
|
||||
self.bytes_written
|
||||
}
|
||||
|
||||
/// flush last block, set END_OF_STREAM flag
|
||||
///
|
||||
/// Note: This may write an empty block just including the
|
||||
/// END_OF_STREAM flag.
|
||||
fn finish(&mut self, incomplete: bool) -> Result<bool, std::io::Error> {
|
||||
vec::clear(&mut self.buffer.payload[self.buffer_pos..]);
|
||||
self.buffer.flags = BlockHeaderFlags::END_OF_STREAM;
|
||||
if incomplete { self.buffer.flags |= BlockHeaderFlags::INCOMPLETE; }
|
||||
self.buffer.set_size(self.buffer_pos);
|
||||
self.buffer.set_seq_nr(self.seq_nr);
|
||||
self.seq_nr += 1;
|
||||
self.bytes_written += BlockHeader::SIZE;
|
||||
Self::write_block(&self.buffer, &mut self.writer)
|
||||
}
|
||||
|
||||
/// Returns if the writer already detected the logical end of media
|
||||
fn logical_end_of_media(&self) -> bool {
|
||||
self.logical_end_of_media
|
||||
}
|
||||
|
||||
}
|
||||
202
src/tape/file_formats/chunk_archive.rs
Normal file
202
src/tape/file_formats/chunk_archive.rs
Normal file
@ -0,0 +1,202 @@
|
||||
use std::io::Read;
|
||||
|
||||
use anyhow::{bail, Error};
|
||||
use endian_trait::Endian;
|
||||
|
||||
use proxmox::tools::{
|
||||
Uuid,
|
||||
io::ReadExt,
|
||||
};
|
||||
|
||||
use crate::backup::DataBlob;
|
||||
|
||||
use crate::tape::{
|
||||
TapeWrite,
|
||||
file_formats::{
|
||||
PROXMOX_TAPE_BLOCK_SIZE,
|
||||
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0,
|
||||
PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
|
||||
MediaContentHeader,
|
||||
ChunkArchiveEntryHeader,
|
||||
},
|
||||
};
|
||||
|
||||
/// Writes chunk archives to tape.
|
||||
///
|
||||
/// A chunk archive consists of a `MediaContentHeader` followed by a
|
||||
/// list of chunks entries. Each chunk entry consists of a
|
||||
/// `ChunkArchiveEntryHeader` folowed by the chunk data (`DataBlob`).
|
||||
///
|
||||
/// `| MediaContentHeader | ( ChunkArchiveEntryHeader | DataBlob )* |`
|
||||
pub struct ChunkArchiveWriter<'a> {
|
||||
writer: Option<Box<dyn TapeWrite + 'a>>,
|
||||
bytes_written: usize, // does not include bytes from current writer
|
||||
close_on_leom: bool,
|
||||
}
|
||||
|
||||
impl <'a> ChunkArchiveWriter<'a> {
|
||||
|
||||
pub const MAGIC: [u8; 8] = PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0;
|
||||
|
||||
/// Creates a new instance
|
||||
pub fn new(mut writer: Box<dyn TapeWrite + 'a>, close_on_leom: bool) -> Result<(Self,Uuid), Error> {
|
||||
|
||||
let header = MediaContentHeader::new(Self::MAGIC, 0);
|
||||
writer.write_header(&header, &[])?;
|
||||
|
||||
let me = Self {
|
||||
writer: Some(writer),
|
||||
bytes_written: 0,
|
||||
close_on_leom,
|
||||
};
|
||||
|
||||
Ok((me, header.uuid.into()))
|
||||
}
|
||||
|
||||
/// Returns the number of bytes written so far.
|
||||
pub fn bytes_written(&self) -> usize {
|
||||
match self.writer {
|
||||
Some(ref writer) => writer.bytes_written(),
|
||||
None => self.bytes_written, // finalize sets this
|
||||
}
|
||||
}
|
||||
|
||||
fn write_all(&mut self, data: &[u8]) -> Result<bool, std::io::Error> {
|
||||
match self.writer {
|
||||
Some(ref mut writer) => writer.write_all(data),
|
||||
None => proxmox::io_bail!(
|
||||
"detected write after archive finished - internal error"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Write chunk into archive.
|
||||
///
|
||||
/// This may return false when `LEOM` is detected (when close_on_leom is set).
|
||||
/// In that case the archive only contains parts of the last chunk.
|
||||
pub fn try_write_chunk(
|
||||
&mut self,
|
||||
digest: &[u8;32],
|
||||
blob: &DataBlob,
|
||||
) -> Result<bool, std::io::Error> {
|
||||
|
||||
if self.writer.is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let head = ChunkArchiveEntryHeader {
|
||||
magic: PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
|
||||
digest: *digest,
|
||||
size: blob.raw_size(),
|
||||
};
|
||||
|
||||
let head = head.to_le();
|
||||
let data = unsafe { std::slice::from_raw_parts(
|
||||
&head as *const ChunkArchiveEntryHeader as *const u8,
|
||||
std::mem::size_of::<ChunkArchiveEntryHeader>())
|
||||
};
|
||||
|
||||
self.write_all(data)?;
|
||||
|
||||
let mut start = 0;
|
||||
let blob_data = blob.raw_data();
|
||||
loop {
|
||||
if start >= blob_data.len() {
|
||||
break;
|
||||
}
|
||||
|
||||
let end = start + PROXMOX_TAPE_BLOCK_SIZE;
|
||||
let mut chunk_is_complete = false;
|
||||
let leom = if end > blob_data.len() {
|
||||
chunk_is_complete = true;
|
||||
self.write_all(&blob_data[start..])?
|
||||
} else {
|
||||
self.write_all(&blob_data[start..end])?
|
||||
};
|
||||
if leom {
|
||||
println!("WRITE DATA LEOM at pos {}", self.bytes_written());
|
||||
if self.close_on_leom {
|
||||
let mut writer = self.writer.take().unwrap();
|
||||
writer.finish(false)?;
|
||||
self.bytes_written = writer.bytes_written();
|
||||
return Ok(chunk_is_complete);
|
||||
}
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// This must be called at the end to add padding and `EOF`
|
||||
///
|
||||
/// Returns true on `LEOM` or when we hit max archive size
|
||||
pub fn finish(&mut self) -> Result<bool, std::io::Error> {
|
||||
match self.writer.take() {
|
||||
Some(mut writer) => {
|
||||
self.bytes_written = writer.bytes_written();
|
||||
writer.finish(false)
|
||||
}
|
||||
None => Ok(true),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read chunk archives.
|
||||
pub struct ChunkArchiveDecoder<R> {
|
||||
reader: R,
|
||||
}
|
||||
|
||||
impl <R: Read> ChunkArchiveDecoder<R> {
|
||||
|
||||
/// Creates a new instance
|
||||
pub fn new(reader: R) -> Self {
|
||||
Self { reader }
|
||||
}
|
||||
|
||||
/// Allow access to the underyling reader
|
||||
pub fn reader(&self) -> &R {
|
||||
&self.reader
|
||||
}
|
||||
|
||||
/// Returns the next chunk (if any).
|
||||
pub fn next_chunk(&mut self) -> Result<Option<([u8;32], DataBlob)>, Error> {
|
||||
|
||||
let mut header = ChunkArchiveEntryHeader {
|
||||
magic: [0u8; 8],
|
||||
digest: [0u8; 32],
|
||||
size: 0,
|
||||
};
|
||||
let data = unsafe {
|
||||
std::slice::from_raw_parts_mut(
|
||||
(&mut header as *mut ChunkArchiveEntryHeader) as *mut u8,
|
||||
std::mem::size_of::<ChunkArchiveEntryHeader>())
|
||||
};
|
||||
|
||||
match self.reader.read_exact_or_eof(data) {
|
||||
Ok(true) => {},
|
||||
Ok(false) => {
|
||||
// last chunk is allowed to be incomplete - simply report EOD
|
||||
return Ok(None);
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
|
||||
if header.magic != PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0 {
|
||||
bail!("wrong magic number");
|
||||
}
|
||||
|
||||
let raw_data = match self.reader.read_exact_allocated(header.size as usize) {
|
||||
Ok(data) => data,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => {
|
||||
// last chunk is allowed to be incomplete - simply report EOD
|
||||
return Ok(None);
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
|
||||
let blob = DataBlob::from_raw(raw_data)?;
|
||||
blob.verify_crc()?;
|
||||
|
||||
Ok(Some((header.digest, blob)))
|
||||
}
|
||||
}
|
||||
276
src/tape/file_formats/mod.rs
Normal file
276
src/tape/file_formats/mod.rs
Normal file
@ -0,0 +1,276 @@
|
||||
//! File format definitions and implementations for data written to
|
||||
//! tapes
|
||||
|
||||
mod blocked_reader;
|
||||
pub use blocked_reader::*;
|
||||
|
||||
mod blocked_writer;
|
||||
pub use blocked_writer::*;
|
||||
|
||||
mod chunk_archive;
|
||||
pub use chunk_archive::*;
|
||||
|
||||
mod snapshot_archive;
|
||||
pub use snapshot_archive::*;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use anyhow::{bail, Error};
|
||||
use ::serde::{Deserialize, Serialize};
|
||||
use endian_trait::Endian;
|
||||
use bitflags::bitflags;
|
||||
|
||||
use proxmox::tools::Uuid;
|
||||
|
||||
use crate::backup::Fingerprint;
|
||||
|
||||
/// We use 256KB blocksize (always)
|
||||
pub const PROXMOX_TAPE_BLOCK_SIZE: usize = 256*1024;
|
||||
|
||||
// openssl::sha::sha256(b"Proxmox Tape Block Header v1.0")[0..8]
|
||||
pub const PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0: [u8; 8] = [220, 189, 175, 202, 235, 160, 165, 40];
|
||||
|
||||
// openssl::sha::sha256(b"Proxmox Backup Content Header v1.0")[0..8];
|
||||
pub const PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0: [u8; 8] = [99, 238, 20, 159, 205, 242, 155, 12];
|
||||
// openssl::sha::sha256(b"Proxmox Backup Tape Label v1.0")[0..8];
|
||||
pub const PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0: [u8; 8] = [42, 5, 191, 60, 176, 48, 170, 57];
|
||||
// openssl::sha::sha256(b"Proxmox Backup MediaSet Label v1.0")
|
||||
pub const PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0: [u8; 8] = [8, 96, 99, 249, 47, 151, 83, 216];
|
||||
|
||||
// openssl::sha::sha256(b"Proxmox Backup Chunk Archive v1.0")[0..8]
|
||||
pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0: [u8; 8] = [62, 173, 167, 95, 49, 76, 6, 110];
|
||||
// openssl::sha::sha256(b"Proxmox Backup Chunk Archive Entry v1.0")[0..8]
|
||||
pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0: [u8; 8] = [72, 87, 109, 242, 222, 66, 143, 220];
|
||||
|
||||
// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.0")[0..8];
|
||||
pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0: [u8; 8] = [9, 182, 2, 31, 125, 232, 114, 133];
|
||||
|
||||
lazy_static::lazy_static!{
|
||||
// Map content magic numbers to human readable names.
|
||||
static ref PROXMOX_TAPE_CONTENT_NAME: HashMap<&'static [u8;8], &'static str> = {
|
||||
let mut map = HashMap::new();
|
||||
map.insert(&PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0, "Proxmox Backup Tape Label v1.0");
|
||||
map.insert(&PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, "Proxmox Backup MediaSet Label v1.0");
|
||||
map.insert(&PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0, "Proxmox Backup Chunk Archive v1.0");
|
||||
map.insert(&PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, "Proxmox Backup Snapshot Archive v1.0");
|
||||
map
|
||||
};
|
||||
}
|
||||
|
||||
/// Map content magic numbers to human readable names.
|
||||
pub fn proxmox_tape_magic_to_text(magic: &[u8; 8]) -> Option<String> {
|
||||
PROXMOX_TAPE_CONTENT_NAME.get(magic).map(|s| String::from(*s))
|
||||
}
|
||||
|
||||
/// Tape Block Header with data payload
|
||||
///
|
||||
/// All tape files are written as sequence of blocks.
|
||||
///
|
||||
/// Note: this struct is large, never put this on the stack!
|
||||
/// so we use an unsized type to avoid that.
|
||||
///
|
||||
/// Tape data block are always read/written with a fixed size
|
||||
/// (`PROXMOX_TAPE_BLOCK_SIZE`). But they may contain less data, so the
|
||||
/// header has an additional size field. For streams of blocks, there
|
||||
/// is a sequence number (`seq_nr`) which may be use for additional
|
||||
/// error checking.
|
||||
#[repr(C,packed)]
|
||||
pub struct BlockHeader {
|
||||
/// fixed value `PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0`
|
||||
pub magic: [u8; 8],
|
||||
pub flags: BlockHeaderFlags,
|
||||
/// size as 3 bytes unsigned, little endian
|
||||
pub size: [u8; 3],
|
||||
/// block sequence number
|
||||
pub seq_nr: u32,
|
||||
pub payload: [u8],
|
||||
}
|
||||
|
||||
bitflags! {
|
||||
/// Header flags (e.g. `END_OF_STREAM` or `INCOMPLETE`)
|
||||
pub struct BlockHeaderFlags: u8 {
|
||||
/// Marks the last block in a stream.
|
||||
const END_OF_STREAM = 0b00000001;
|
||||
/// Mark multivolume streams (when set in the last block)
|
||||
const INCOMPLETE = 0b00000010;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Endian, Copy, Clone, Debug)]
|
||||
#[repr(C,packed)]
|
||||
/// Media Content Header
|
||||
///
|
||||
/// All tape files start with this header. The header may contain some
|
||||
/// informational data indicated by `size`.
|
||||
///
|
||||
/// `| MediaContentHeader | header data (size) | stream data |`
|
||||
///
|
||||
/// Note: The stream data following may be of any size.
|
||||
pub struct MediaContentHeader {
|
||||
/// fixed value `PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0`
|
||||
pub magic: [u8; 8],
|
||||
/// magic number for the content following
|
||||
pub content_magic: [u8; 8],
|
||||
/// unique ID to identify this data stream
|
||||
pub uuid: [u8; 16],
|
||||
/// stream creation time
|
||||
pub ctime: i64,
|
||||
/// Size of header data
|
||||
pub size: u32,
|
||||
/// Part number for multipart archives.
|
||||
pub part_number: u8,
|
||||
/// Reserved for future use
|
||||
pub reserved_0: u8,
|
||||
/// Reserved for future use
|
||||
pub reserved_1: u8,
|
||||
/// Reserved for future use
|
||||
pub reserved_2: u8,
|
||||
}
|
||||
|
||||
impl MediaContentHeader {
|
||||
|
||||
/// Create a new instance with autogenerated Uuid
|
||||
pub fn new(content_magic: [u8; 8], size: u32) -> Self {
|
||||
let uuid = *proxmox::tools::uuid::Uuid::generate()
|
||||
.into_inner();
|
||||
Self {
|
||||
magic: PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
|
||||
content_magic,
|
||||
uuid,
|
||||
ctime: proxmox::tools::time::epoch_i64(),
|
||||
size,
|
||||
part_number: 0,
|
||||
reserved_0: 0,
|
||||
reserved_1: 0,
|
||||
reserved_2: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper to check magic numbers and size constraints
|
||||
pub fn check(&self, content_magic: [u8; 8], min_size: u32, max_size: u32) -> Result<(), Error> {
|
||||
if self.magic != PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0 {
|
||||
bail!("MediaContentHeader: wrong magic");
|
||||
}
|
||||
if self.content_magic != content_magic {
|
||||
bail!("MediaContentHeader: wrong content magic");
|
||||
}
|
||||
if self.size < min_size || self.size > max_size {
|
||||
bail!("MediaContentHeader: got unexpected size");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the content Uuid
|
||||
pub fn content_uuid(&self) -> Uuid {
|
||||
Uuid::from(self.uuid)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Endian)]
|
||||
#[repr(C,packed)]
|
||||
/// Header for data blobs inside a chunk archive
|
||||
pub struct ChunkArchiveEntryHeader {
|
||||
/// fixed value `PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0`
|
||||
pub magic: [u8; 8],
|
||||
/// Chunk digest
|
||||
pub digest: [u8; 32],
|
||||
/// Chunk size
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize,Deserialize,Clone,Debug)]
|
||||
/// Media Label
|
||||
///
|
||||
/// Media labels are used to uniquely identify a media. They are
|
||||
/// stored as first file on the tape.
|
||||
pub struct MediaLabel {
|
||||
/// Unique ID
|
||||
pub uuid: Uuid,
|
||||
/// Media label text (or Barcode)
|
||||
pub label_text: String,
|
||||
/// Creation time stamp
|
||||
pub ctime: i64,
|
||||
}
|
||||
|
||||
|
||||
#[derive(Serialize,Deserialize,Clone,Debug)]
|
||||
/// `MediaSet` Label
|
||||
///
|
||||
/// Used to uniquely identify a `MediaSet`. They are stored as second
|
||||
/// file on the tape.
|
||||
pub struct MediaSetLabel {
|
||||
/// The associated `MediaPool`
|
||||
pub pool: String,
|
||||
/// Uuid. We use the all-zero Uuid to reseve an empty media for a specific pool
|
||||
pub uuid: Uuid,
|
||||
/// Media sequence number
|
||||
pub seq_nr: u64,
|
||||
/// Creation time stamp
|
||||
pub ctime: i64,
|
||||
/// Encryption key finkerprint (if encryped)
|
||||
#[serde(skip_serializing_if="Option::is_none")]
|
||||
pub encryption_key_fingerprint: Option<Fingerprint>,
|
||||
}
|
||||
|
||||
impl MediaSetLabel {
|
||||
|
||||
pub fn with_data(
|
||||
pool: &str,
|
||||
uuid: Uuid,
|
||||
seq_nr: u64,
|
||||
ctime: i64,
|
||||
encryption_key_fingerprint: Option<Fingerprint>,
|
||||
) -> Self {
|
||||
Self {
|
||||
pool: pool.to_string(),
|
||||
uuid,
|
||||
seq_nr,
|
||||
ctime,
|
||||
encryption_key_fingerprint,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BlockHeader {
|
||||
|
||||
pub const SIZE: usize = PROXMOX_TAPE_BLOCK_SIZE;
|
||||
|
||||
/// Allocates a new instance on the heap
|
||||
pub fn new() -> Box<Self> {
|
||||
use std::alloc::{alloc_zeroed, Layout};
|
||||
|
||||
let mut buffer = unsafe {
|
||||
let ptr = alloc_zeroed(
|
||||
Layout::from_size_align(Self::SIZE, std::mem::align_of::<u64>())
|
||||
.unwrap(),
|
||||
);
|
||||
Box::from_raw(
|
||||
std::slice::from_raw_parts_mut(ptr, Self::SIZE - 16)
|
||||
as *mut [u8] as *mut Self
|
||||
)
|
||||
};
|
||||
buffer.magic = PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0;
|
||||
buffer
|
||||
}
|
||||
|
||||
/// Set the `size` field
|
||||
pub fn set_size(&mut self, size: usize) {
|
||||
let size = size.to_le_bytes();
|
||||
self.size.copy_from_slice(&size[..3]);
|
||||
}
|
||||
|
||||
/// Returns the `size` field
|
||||
pub fn size(&self) -> usize {
|
||||
(self.size[0] as usize) + ((self.size[1] as usize)<<8) + ((self.size[2] as usize)<<16)
|
||||
}
|
||||
|
||||
/// Set the `seq_nr` field
|
||||
pub fn set_seq_nr(&mut self, seq_nr: u32) {
|
||||
self.seq_nr = seq_nr.to_le();
|
||||
}
|
||||
|
||||
/// Returns the `seq_nr` field
|
||||
pub fn seq_nr(&self) -> u32 {
|
||||
u32::from_le(self.seq_nr)
|
||||
}
|
||||
}
|
||||
139
src/tape/file_formats/snapshot_archive.rs
Normal file
139
src/tape/file_formats/snapshot_archive.rs
Normal file
@ -0,0 +1,139 @@
|
||||
use std::io::{Read, Write};
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use proxmox::{
|
||||
sys::error::SysError,
|
||||
tools::Uuid,
|
||||
};
|
||||
|
||||
use crate::tape::{
|
||||
TapeWrite,
|
||||
SnapshotReader,
|
||||
file_formats::{
|
||||
PROXMOX_TAPE_BLOCK_SIZE,
|
||||
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
|
||||
MediaContentHeader,
|
||||
},
|
||||
};
|
||||
|
||||
/// Write a set of files as `pxar` archive to the tape
|
||||
///
|
||||
/// This ignores file attributes like ACLs and xattrs.
|
||||
///
|
||||
/// Returns `Ok(Some(content_uuid))` on succees, and `Ok(None)` if
|
||||
/// `LEOM` was detected before all data was written. The stream is
|
||||
/// marked inclomplete in that case and does not contain all data (The
|
||||
/// backup task must rewrite the whole file on the next media).
|
||||
pub fn tape_write_snapshot_archive<'a>(
|
||||
writer: &mut (dyn TapeWrite + 'a),
|
||||
snapshot_reader: &SnapshotReader,
|
||||
) -> Result<Option<Uuid>, std::io::Error> {
|
||||
|
||||
let snapshot = snapshot_reader.snapshot().to_string();
|
||||
let file_list = snapshot_reader.file_list();
|
||||
|
||||
let header_data = snapshot.as_bytes().to_vec();
|
||||
|
||||
let header = MediaContentHeader::new(
|
||||
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, header_data.len() as u32);
|
||||
let content_uuid = header.uuid.into();
|
||||
|
||||
let root_metadata = pxar::Metadata::dir_builder(0o0664).build();
|
||||
|
||||
let mut file_copy_buffer = proxmox::tools::vec::undefined(PROXMOX_TAPE_BLOCK_SIZE);
|
||||
|
||||
let result: Result<(), std::io::Error> = proxmox::try_block!({
|
||||
|
||||
let leom = writer.write_header(&header, &header_data)?;
|
||||
if leom {
|
||||
return Err(std::io::Error::from_raw_os_error(nix::errno::Errno::ENOSPC as i32));
|
||||
}
|
||||
|
||||
let mut encoder = pxar::encoder::sync::Encoder::new(PxarTapeWriter::new(writer), &root_metadata)?;
|
||||
|
||||
for filename in file_list.iter() {
|
||||
|
||||
let mut file = snapshot_reader.open_file(filename)
|
||||
.map_err(|err| proxmox::io_format_err!("open file '{}' failed - {}", filename, err))?;
|
||||
let metadata = file.metadata()?;
|
||||
let file_size = metadata.len();
|
||||
|
||||
let metadata: pxar::Metadata = metadata.into();
|
||||
|
||||
if !metadata.is_regular_file() {
|
||||
proxmox::io_bail!("file '{}' is not a regular file", filename);
|
||||
}
|
||||
|
||||
let mut remaining = file_size;
|
||||
let mut out = encoder.create_file(&metadata, filename, file_size)?;
|
||||
while remaining != 0 {
|
||||
let got = file.read(&mut file_copy_buffer[..])?;
|
||||
if got as u64 > remaining {
|
||||
proxmox::io_bail!("file '{}' changed while reading", filename);
|
||||
}
|
||||
out.write_all(&file_copy_buffer[..got])?;
|
||||
remaining -= got as u64;
|
||||
|
||||
}
|
||||
if remaining > 0 {
|
||||
proxmox::io_bail!("file '{}' shrunk while reading", filename);
|
||||
}
|
||||
}
|
||||
encoder.finish()?;
|
||||
Ok(())
|
||||
});
|
||||
|
||||
match result {
|
||||
Ok(()) => {
|
||||
writer.finish(false)?;
|
||||
Ok(Some(content_uuid))
|
||||
}
|
||||
Err(err) => {
|
||||
if err.is_errno(nix::errno::Errno::ENOSPC) && writer.logical_end_of_media() {
|
||||
writer.finish(true)?; // mark as incomplete
|
||||
Ok(None)
|
||||
} else {
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helper to create pxar archives on tape
|
||||
//
|
||||
// We generate and error at LEOM,
|
||||
struct PxarTapeWriter<'a, T: TapeWrite + ?Sized> {
|
||||
inner: &'a mut T,
|
||||
}
|
||||
|
||||
impl<'a, T: TapeWrite + ?Sized> PxarTapeWriter<'a, T> {
|
||||
pub fn new(inner: &'a mut T) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, T: TapeWrite + ?Sized> pxar::encoder::SeqWrite for PxarTapeWriter<'a, T> {
|
||||
|
||||
fn poll_seq_write(
|
||||
self: Pin<&mut Self>,
|
||||
_cx: &mut Context,
|
||||
buf: &[u8],
|
||||
) -> Poll<std::io::Result<usize>> {
|
||||
let this = unsafe { self.get_unchecked_mut() };
|
||||
Poll::Ready(match this.inner.write_all(buf) {
|
||||
Ok(leom) => {
|
||||
if leom {
|
||||
Err(std::io::Error::from_raw_os_error(nix::errno::Errno::ENOSPC as i32))
|
||||
} else {
|
||||
Ok(buf.len())
|
||||
}
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
})
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user