From 018d11bb84dedd33f2cdbce46216eed35b58da08 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 14 Aug 2019 13:05:11 +0200 Subject: [PATCH] src/backup/data_blob.rs: move parts into single files --- src/backup.rs | 18 + src/backup/checksum_reader.rs | 47 +++ src/backup/checksum_writer.rs | 48 +++ src/backup/crypt_reader.rs | 90 +++++ src/backup/crypt_writer.rs | 63 ++++ src/backup/data_blob.rs | 643 --------------------------------- src/backup/data_blob_reader.rs | 169 +++++++++ src/backup/data_blob_writer.rs | 244 +++++++++++++ 8 files changed, 679 insertions(+), 643 deletions(-) create mode 100644 src/backup/checksum_reader.rs create mode 100644 src/backup/checksum_writer.rs create mode 100644 src/backup/crypt_reader.rs create mode 100644 src/backup/crypt_writer.rs create mode 100644 src/backup/data_blob_reader.rs create mode 100644 src/backup/data_blob_writer.rs diff --git a/src/backup.rs b/src/backup.rs index 5cca5885..a9d03e01 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -121,12 +121,30 @@ pub use crypt_config::*; mod key_derivation; pub use key_derivation::*; +mod crypt_reader; +pub use crypt_reader::*; + +mod crypt_writer; +pub use crypt_writer::*; + +mod checksum_reader; +pub use checksum_reader::*; + +mod checksum_writer; +pub use checksum_writer::*; + mod data_chunk; pub use data_chunk::*; mod data_blob; pub use data_blob::*; +mod data_blob_reader; +pub use data_blob_reader::*; + +mod data_blob_writer; +pub use data_blob_writer::*; + mod chunk_stream; pub use chunk_stream::*; diff --git a/src/backup/checksum_reader.rs b/src/backup/checksum_reader.rs new file mode 100644 index 00000000..da8cf702 --- /dev/null +++ b/src/backup/checksum_reader.rs @@ -0,0 +1,47 @@ +use failure::*; +use std::io::Read; + +pub struct ChecksumReader<'a, R> { + reader: R, + hasher: crc32fast::Hasher, + signer: Option>, +} + +impl <'a, R: Read> ChecksumReader<'a, R> { + + pub fn new(reader: R, signer: Option>) -> Self { + let hasher = crc32fast::Hasher::new(); + Self { reader, hasher, signer } + } + + pub fn finish(mut self) -> Result<(R, u32, Option<[u8; 32]>), Error> { + let crc = self.hasher.finalize(); + + if let Some(ref mut signer) = self.signer { + let mut tag = [0u8; 32]; + signer.sign(&mut tag)?; + Ok((self.reader, crc, Some(tag))) + } else { + Ok((self.reader, crc, None)) + } + } +} + +impl <'a, R: Read> Read for ChecksumReader<'a, R> { + + fn read(&mut self, buf: &mut [u8]) -> Result { + let count = self.reader.read(buf)?; + if count > 0 { + self.hasher.update(&buf[..count]); + if let Some(ref mut signer) = self.signer { + signer.update(&buf[..count]) + .map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::Other, + format!("hmac update failed - {}", err)) + })?; + } + } + Ok(count) + } +} diff --git a/src/backup/checksum_writer.rs b/src/backup/checksum_writer.rs new file mode 100644 index 00000000..f3bbe30c --- /dev/null +++ b/src/backup/checksum_writer.rs @@ -0,0 +1,48 @@ +use failure::*; +use std::io::Write; + +pub struct ChecksumWriter<'a, W> { + writer: W, + hasher: crc32fast::Hasher, + signer: Option>, +} + +impl <'a, W: Write> ChecksumWriter<'a, W> { + + pub fn new(writer: W, signer: Option>) -> Self { + let hasher = crc32fast::Hasher::new(); + Self { writer, hasher, signer } + } + + pub fn finish(mut self) -> Result<(W, u32, Option<[u8; 32]>), Error> { + let crc = self.hasher.finalize(); + + if let Some(ref mut signer) = self.signer { + let mut tag = [0u8; 32]; + signer.sign(&mut tag)?; + Ok((self.writer, crc, Some(tag))) + } else { + Ok((self.writer, crc, None)) + } + } +} + +impl <'a, W: Write> Write for ChecksumWriter<'a, W> { + + fn write(&mut self, buf: &[u8]) -> Result { + self.hasher.update(buf); + if let Some(ref mut signer) = self.signer { + signer.update(buf) + .map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::Other, + format!("hmac update failed - {}", err)) + })?; + } + self.writer.write(buf) + } + + fn flush(&mut self) -> Result<(), std::io::Error> { + self.writer.flush() + } +} diff --git a/src/backup/crypt_reader.rs b/src/backup/crypt_reader.rs new file mode 100644 index 00000000..c36c8b66 --- /dev/null +++ b/src/backup/crypt_reader.rs @@ -0,0 +1,90 @@ +use failure::*; +use std::io::{Read, BufRead}; + +use super::CryptConfig; + +pub struct CryptReader { + reader: R, + small_read_buf: Vec, + block_size: usize, + crypter: openssl::symm::Crypter, + finalized: bool, +} + +impl CryptReader { + + pub fn new(reader: R, iv: [u8; 16], tag: [u8; 16], config: &CryptConfig) -> Result { + let block_size = config.cipher().block_size(); // Note: block size is normally 1 byte for stream ciphers + if block_size.count_ones() != 1 || block_size > 512 { + bail!("unexpected Cipher block size {}", block_size); + } + let mut crypter = config.data_crypter(&iv, openssl::symm::Mode::Decrypt)?; + crypter.set_tag(&tag)?; + + Ok(Self { reader, crypter, block_size, finalized: false, small_read_buf: Vec::new() }) + } + + pub fn finish(self) -> Result { + if !self.finalized { + bail!("CryptReader not successfully finalized."); + } + Ok(self.reader) + } +} + +impl Read for CryptReader { + + fn read(&mut self, buf: &mut [u8]) -> Result { + if self.small_read_buf.len() > 0 { + let max = if self.small_read_buf.len() > buf.len() { buf.len() } else { self.small_read_buf.len() }; + let rest = self.small_read_buf.split_off(max); + buf[..max].copy_from_slice(&self.small_read_buf); + self.small_read_buf = rest; + return Ok(max); + } + + let data = self.reader.fill_buf()?; + + // handle small read buffers + if buf.len() <= 2*self.block_size { + let mut outbuf = [0u8; 1024]; + + let count = if data.len() == 0 { // EOF + let written = self.crypter.finalize(&mut outbuf)?; + self.finalized = true; + written + } else { + let mut read_size = outbuf.len() - self.block_size; + if read_size > data.len() { + read_size = data.len(); + } + let written = self.crypter.update(&data[..read_size], &mut outbuf)?; + self.reader.consume(read_size); + written + }; + + if count > buf.len() { + buf.copy_from_slice(&outbuf[..buf.len()]); + self.small_read_buf = outbuf[buf.len()..count].to_vec(); + return Ok(buf.len()); + } else { + buf[..count].copy_from_slice(&outbuf[..count]); + return Ok(count); + } + } else { + if data.len() == 0 { // EOF + let rest = self.crypter.finalize(buf)?; + self.finalized = true; + return Ok(rest) + } else { + let mut read_size = buf.len() - self.block_size; + if read_size > data.len() { + read_size = data.len(); + } + let count = self.crypter.update(&data[..read_size], buf)?; + self.reader.consume(read_size); + return Ok(count) + } + } + } +} diff --git a/src/backup/crypt_writer.rs b/src/backup/crypt_writer.rs new file mode 100644 index 00000000..06a6aed9 --- /dev/null +++ b/src/backup/crypt_writer.rs @@ -0,0 +1,63 @@ +use failure::*; +use std::io::Write; + +use super::CryptConfig; + +pub struct CryptWriter { + writer: W, + block_size: usize, + encr_buf: Box<[u8; 64*1024]>, + iv: [u8; 16], + crypter: openssl::symm::Crypter, +} + +impl CryptWriter { + + pub fn new(writer: W, config: &CryptConfig) -> Result { + let mut iv = [0u8; 16]; + proxmox::sys::linux::fill_with_random_data(&mut iv)?; + let block_size = config.cipher().block_size(); + + let crypter = config.data_crypter(&iv, openssl::symm::Mode::Encrypt)?; + + Ok(Self { writer, iv, crypter, block_size, encr_buf: Box::new([0u8; 64*1024]) }) + } + + pub fn finish(mut self) -> Result<(W, [u8; 16], [u8; 16]), Error> { + let rest = self.crypter.finalize(self.encr_buf.as_mut())?; + if rest > 0 { + self.writer.write_all(&self.encr_buf[..rest])?; + } + + self.writer.flush()?; + + let mut tag = [0u8; 16]; + self.crypter.get_tag(&mut tag)?; + + Ok((self.writer, self.iv, tag)) + } +} + +impl Write for CryptWriter { + + fn write(&mut self, buf: &[u8]) -> Result { + let mut write_size = buf.len(); + if write_size > (self.encr_buf.len() - self.block_size) { + write_size = self.encr_buf.len() - self.block_size; + } + let count = self.crypter.update(&buf[..write_size], self.encr_buf.as_mut()) + .map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::Other, + format!("crypter update failed - {}", err)) + })?; + + self.writer.write_all(&self.encr_buf[..count])?; + + Ok(write_size) + } + + fn flush(&mut self) -> Result<(), std::io::Error> { + Ok(()) + } +} diff --git a/src/backup/data_blob.rs b/src/backup/data_blob.rs index 63e7acc2..29337d98 100644 --- a/src/backup/data_blob.rs +++ b/src/backup/data_blob.rs @@ -7,7 +7,6 @@ const MAX_BLOB_SIZE: usize = 128*1024*1024; use super::*; - /// Data blob binary storage format /// /// Data blobs store arbitrary binary data (< 128MB), and can be @@ -288,646 +287,4 @@ impl DataBlob { bail!("unable to parse raw blob - wrong magic"); } } - -} - -use std::io::{Read, BufRead, BufReader, Write, Seek, SeekFrom}; - -struct CryptReader { - reader: R, - small_read_buf: Vec, - block_size: usize, - crypter: openssl::symm::Crypter, - finalized: bool, -} - -impl CryptReader { - - fn new(reader: R, iv: [u8; 16], tag: [u8; 16], config: &CryptConfig) -> Result { - let block_size = config.cipher().block_size(); // Note: block size is normally 1 byte for stream ciphers - if block_size.count_ones() != 1 || block_size > 512 { - bail!("unexpected Cipher block size {}", block_size); - } - let mut crypter = config.data_crypter(&iv, openssl::symm::Mode::Decrypt)?; - crypter.set_tag(&tag)?; - - Ok(Self { reader, crypter, block_size, finalized: false, small_read_buf: Vec::new() }) - } - - fn finish(self) -> Result { - if !self.finalized { - bail!("CryptReader not successfully finalized."); - } - Ok(self.reader) - } -} - -impl Read for CryptReader { - - fn read(&mut self, buf: &mut [u8]) -> Result { - if self.small_read_buf.len() > 0 { - let max = if self.small_read_buf.len() > buf.len() { buf.len() } else { self.small_read_buf.len() }; - let rest = self.small_read_buf.split_off(max); - buf[..max].copy_from_slice(&self.small_read_buf); - self.small_read_buf = rest; - return Ok(max); - } - - let data = self.reader.fill_buf()?; - - // handle small read buffers - if buf.len() <= 2*self.block_size { - let mut outbuf = [0u8; 1024]; - - let count = if data.len() == 0 { // EOF - let written = self.crypter.finalize(&mut outbuf)?; - self.finalized = true; - written - } else { - let mut read_size = outbuf.len() - self.block_size; - if read_size > data.len() { - read_size = data.len(); - } - let written = self.crypter.update(&data[..read_size], &mut outbuf)?; - self.reader.consume(read_size); - written - }; - - if count > buf.len() { - buf.copy_from_slice(&outbuf[..buf.len()]); - self.small_read_buf = outbuf[buf.len()..count].to_vec(); - return Ok(buf.len()); - } else { - buf[..count].copy_from_slice(&outbuf[..count]); - return Ok(count); - } - } else { - if data.len() == 0 { // EOF - let rest = self.crypter.finalize(buf)?; - self.finalized = true; - return Ok(rest) - } else { - let mut read_size = buf.len() - self.block_size; - if read_size > data.len() { - read_size = data.len(); - } - let count = self.crypter.update(&data[..read_size], buf)?; - self.reader.consume(read_size); - return Ok(count) - } - } - } -} - -struct CryptWriter { - writer: W, - block_size: usize, - encr_buf: Box<[u8; 64*1024]>, - iv: [u8; 16], - crypter: openssl::symm::Crypter, -} - -impl CryptWriter { - - fn new(writer: W, config: &CryptConfig) -> Result { - let mut iv = [0u8; 16]; - proxmox::sys::linux::fill_with_random_data(&mut iv)?; - let block_size = config.cipher().block_size(); - - let crypter = config.data_crypter(&iv, openssl::symm::Mode::Encrypt)?; - - Ok(Self { writer, iv, crypter, block_size, encr_buf: Box::new([0u8; 64*1024]) }) - } - - fn finish(mut self) -> Result<(W, [u8; 16], [u8; 16]), Error> { - let rest = self.crypter.finalize(self.encr_buf.as_mut())?; - if rest > 0 { - self.writer.write_all(&self.encr_buf[..rest])?; - } - - self.writer.flush()?; - - let mut tag = [0u8; 16]; - self.crypter.get_tag(&mut tag)?; - - Ok((self.writer, self.iv, tag)) - } -} - -impl Write for CryptWriter { - - fn write(&mut self, buf: &[u8]) -> Result { - let mut write_size = buf.len(); - if write_size > (self.encr_buf.len() - self.block_size) { - write_size = self.encr_buf.len() - self.block_size; - } - let count = self.crypter.update(&buf[..write_size], self.encr_buf.as_mut()) - .map_err(|err| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("crypter update failed - {}", err)) - })?; - - self.writer.write_all(&self.encr_buf[..count])?; - - Ok(write_size) - } - - fn flush(&mut self) -> Result<(), std::io::Error> { - Ok(()) - } -} - -struct ChecksumWriter<'a, W> { - writer: W, - hasher: crc32fast::Hasher, - signer: Option>, -} - -impl <'a, W: Write> ChecksumWriter<'a, W> { - - fn new(writer: W, signer: Option>) -> Self { - let hasher = crc32fast::Hasher::new(); - Self { writer, hasher, signer } - } - - pub fn finish(mut self) -> Result<(W, u32, Option<[u8; 32]>), Error> { - let crc = self.hasher.finalize(); - - if let Some(ref mut signer) = self.signer { - let mut tag = [0u8; 32]; - signer.sign(&mut tag)?; - Ok((self.writer, crc, Some(tag))) - } else { - Ok((self.writer, crc, None)) - } - } -} - -impl <'a, W: Write> Write for ChecksumWriter<'a, W> { - - fn write(&mut self, buf: &[u8]) -> Result { - self.hasher.update(buf); - if let Some(ref mut signer) = self.signer { - signer.update(buf) - .map_err(|err| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("hmac update failed - {}", err)) - })?; - } - self.writer.write(buf) - } - - fn flush(&mut self) -> Result<(), std::io::Error> { - self.writer.flush() - } -} - -enum BlobWriterState<'a, W: Write> { - Uncompressed { csum_writer: ChecksumWriter<'a, W> }, - Compressed { compr: zstd::stream::write::Encoder> }, - Signed { csum_writer: ChecksumWriter<'a, W> }, - SignedCompressed { compr: zstd::stream::write::Encoder> }, - Encrypted { crypt_writer: CryptWriter> }, - EncryptedCompressed { compr: zstd::stream::write::Encoder>> }, -} - -/// Write compressed data blobs -pub struct DataBlobWriter<'a, W: Write> { - state: BlobWriterState<'a, W>, -} - -impl <'a, W: Write + Seek> DataBlobWriter<'a, W> { - - pub fn new_uncompressed(mut writer: W) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = DataBlobHeader { magic: UNCOMPRESSED_BLOB_MAGIC_1_0, crc: [0; 4] }; - unsafe { - writer.write_le_value(head)?; - } - let csum_writer = ChecksumWriter::new(writer, None); - Ok(Self { state: BlobWriterState::Uncompressed { csum_writer }}) - } - - pub fn new_compressed(mut writer: W) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = DataBlobHeader { magic: COMPRESSED_BLOB_MAGIC_1_0, crc: [0; 4] }; - unsafe { - writer.write_le_value(head)?; - } - let csum_writer = ChecksumWriter::new(writer, None); - let compr = zstd::stream::write::Encoder::new(csum_writer, 1)?; - Ok(Self { state: BlobWriterState::Compressed { compr }}) - } - - pub fn new_signed(mut writer: W, config: &'a CryptConfig) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = AuthenticatedDataBlobHeader { - head: DataBlobHeader { magic: AUTHENTICATED_BLOB_MAGIC_1_0, crc: [0; 4] }, - tag: [0u8; 32], - }; - unsafe { - writer.write_le_value(head)?; - } - let signer = config.data_signer(); - let csum_writer = ChecksumWriter::new(writer, Some(signer)); - Ok(Self { state: BlobWriterState::Signed { csum_writer }}) - } - - pub fn new_signed_compressed(mut writer: W, config: &'a CryptConfig) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = AuthenticatedDataBlobHeader { - head: DataBlobHeader { magic: AUTH_COMPR_BLOB_MAGIC_1_0, crc: [0; 4] }, - tag: [0u8; 32], - }; - unsafe { - writer.write_le_value(head)?; - } - let signer = config.data_signer(); - let csum_writer = ChecksumWriter::new(writer, Some(signer)); - let compr = zstd::stream::write::Encoder::new(csum_writer, 1)?; - Ok(Self { state: BlobWriterState::SignedCompressed { compr }}) - } - - pub fn new_encrypted(mut writer: W, config: &'a CryptConfig) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { magic: ENCRYPTED_BLOB_MAGIC_1_0, crc: [0; 4] }, - iv: [0u8; 16], - tag: [0u8; 16], - }; - unsafe { - writer.write_le_value(head)?; - } - - let csum_writer = ChecksumWriter::new(writer, None); - let crypt_writer = CryptWriter::new(csum_writer, config)?; - Ok(Self { state: BlobWriterState::Encrypted { crypt_writer }}) - } - - pub fn new_encrypted_compressed(mut writer: W, config: &'a CryptConfig) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { magic: ENCR_COMPR_BLOB_MAGIC_1_0, crc: [0; 4] }, - iv: [0u8; 16], - tag: [0u8; 16], - }; - unsafe { - writer.write_le_value(head)?; - } - - let csum_writer = ChecksumWriter::new(writer, None); - let crypt_writer = CryptWriter::new(csum_writer, config)?; - let compr = zstd::stream::write::Encoder::new(crypt_writer, 1)?; - Ok(Self { state: BlobWriterState::EncryptedCompressed { compr }}) - } - - pub fn finish(self) -> Result { - match self.state { - BlobWriterState::Uncompressed { csum_writer } => { - // write CRC - let (mut writer, crc, _) = csum_writer.finish()?; - let head = DataBlobHeader { magic: UNCOMPRESSED_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }; - - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - - return Ok(writer) - } - BlobWriterState::Compressed { compr } => { - let csum_writer = compr.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = DataBlobHeader { magic: COMPRESSED_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }; - - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - - return Ok(writer) - } - BlobWriterState::Signed { csum_writer } => { - let (mut writer, crc, tag) = csum_writer.finish()?; - - let head = AuthenticatedDataBlobHeader { - head: DataBlobHeader { magic: AUTHENTICATED_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }, - tag: tag.unwrap(), - }; - - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - - return Ok(writer) - } - BlobWriterState::SignedCompressed { compr } => { - let csum_writer = compr.finish()?; - let (mut writer, crc, tag) = csum_writer.finish()?; - - let head = AuthenticatedDataBlobHeader { - head: DataBlobHeader { magic: AUTH_COMPR_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }, - tag: tag.unwrap(), - }; - - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - - return Ok(writer) - } - BlobWriterState::Encrypted { crypt_writer } => { - let (csum_writer, iv, tag) = crypt_writer.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { magic: ENCRYPTED_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }, - iv, tag, - }; - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - return Ok(writer) - } - BlobWriterState::EncryptedCompressed { compr } => { - let crypt_writer = compr.finish()?; - let (csum_writer, iv, tag) = crypt_writer.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { magic: ENCR_COMPR_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }, - iv, tag, - }; - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - return Ok(writer) - } - } - } -} - -impl <'a, W: Write + Seek> Write for DataBlobWriter<'a, W> { - - fn write(&mut self, buf: &[u8]) -> Result { - match self.state { - BlobWriterState::Uncompressed { ref mut csum_writer } => { - csum_writer.write(buf) - } - BlobWriterState::Compressed { ref mut compr } => { - compr.write(buf) - } - BlobWriterState::Signed { ref mut csum_writer } => { - csum_writer.write(buf) - } - BlobWriterState::SignedCompressed { ref mut compr } => { - compr.write(buf) - } - BlobWriterState::Encrypted { ref mut crypt_writer } => { - crypt_writer.write(buf) - } - BlobWriterState::EncryptedCompressed { ref mut compr } => { - compr.write(buf) - } - } - } - - fn flush(&mut self) -> Result<(), std::io::Error> { - match self.state { - BlobWriterState::Uncompressed { ref mut csum_writer } => { - csum_writer.flush() - } - BlobWriterState::Compressed { ref mut compr } => { - compr.flush() - } - BlobWriterState::Signed { ref mut csum_writer } => { - csum_writer.flush() - } - BlobWriterState::SignedCompressed { ref mut compr } => { - compr.flush() - } - BlobWriterState::Encrypted { ref mut crypt_writer } => { - crypt_writer.flush() - } - BlobWriterState::EncryptedCompressed { ref mut compr } => { - compr.flush() - } - } - } -} - -struct ChecksumReader<'a, R> { - reader: R, - hasher: crc32fast::Hasher, - signer: Option>, -} - -impl <'a, R: Read> ChecksumReader<'a, R> { - - fn new(reader: R, signer: Option>) -> Self { - let hasher = crc32fast::Hasher::new(); - Self { reader, hasher, signer } - } - - pub fn finish(mut self) -> Result<(R, u32, Option<[u8; 32]>), Error> { - let crc = self.hasher.finalize(); - - if let Some(ref mut signer) = self.signer { - let mut tag = [0u8; 32]; - signer.sign(&mut tag)?; - Ok((self.reader, crc, Some(tag))) - } else { - Ok((self.reader, crc, None)) - } - } -} - -impl <'a, R: Read> Read for ChecksumReader<'a, R> { - - fn read(&mut self, buf: &mut [u8]) -> Result { - let count = self.reader.read(buf)?; - if count > 0 { - self.hasher.update(&buf[..count]); - if let Some(ref mut signer) = self.signer { - signer.update(&buf[..count]) - .map_err(|err| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("hmac update failed - {}", err)) - })?; - } - } - Ok(count) - } -} - -enum BlobReaderState<'a, R: Read> { - Uncompressed { expected_crc: u32, csum_reader: ChecksumReader<'a, R> }, - Compressed { expected_crc: u32, decompr: zstd::stream::read::Decoder>> }, - Signed { expected_crc: u32, expected_hmac: [u8; 32], csum_reader: ChecksumReader<'a, R> }, - SignedCompressed { expected_crc: u32, expected_hmac: [u8; 32], decompr: zstd::stream::read::Decoder>> }, - Encrypted { expected_crc: u32, decrypt_reader: CryptReader>> }, - EncryptedCompressed { expected_crc: u32, decompr: zstd::stream::read::Decoder>>>> }, -} - -/// Read data blobs -pub struct DataBlobReader<'a, R: Read> { - state: BlobReaderState<'a, R>, -} - -impl <'a, R: Read> DataBlobReader<'a, R> { - - pub fn new(mut reader: R, config: Option<&'a CryptConfig>) -> Result { - - let head: DataBlobHeader = unsafe { reader.read_le_value()? }; - match head.magic { - UNCOMPRESSED_BLOB_MAGIC_1_0 => { - let expected_crc = u32::from_le_bytes(head.crc); - let csum_reader = ChecksumReader::new(reader, None); - Ok(Self { state: BlobReaderState::Uncompressed { expected_crc, csum_reader }}) - } - COMPRESSED_BLOB_MAGIC_1_0 => { - let expected_crc = u32::from_le_bytes(head.crc); - let csum_reader = ChecksumReader::new(reader, None); - - let decompr = zstd::stream::read::Decoder::new(csum_reader)?; - Ok(Self { state: BlobReaderState::Compressed { expected_crc, decompr }}) - } - AUTHENTICATED_BLOB_MAGIC_1_0 => { - let expected_crc = u32::from_le_bytes(head.crc); - let mut expected_hmac = [0u8; 32]; - reader.read_exact(&mut expected_hmac)?; - let signer = config.map(|c| c.data_signer()); - let csum_reader = ChecksumReader::new(reader, signer); - Ok(Self { state: BlobReaderState::Signed { expected_crc, expected_hmac, csum_reader }}) - } - AUTH_COMPR_BLOB_MAGIC_1_0 => { - let expected_crc = u32::from_le_bytes(head.crc); - let mut expected_hmac = [0u8; 32]; - reader.read_exact(&mut expected_hmac)?; - let signer = config.map(|c| c.data_signer()); - let csum_reader = ChecksumReader::new(reader, signer); - - let decompr = zstd::stream::read::Decoder::new(csum_reader)?; - Ok(Self { state: BlobReaderState::SignedCompressed { expected_crc, expected_hmac, decompr }}) - } - ENCRYPTED_BLOB_MAGIC_1_0 => { - let expected_crc = u32::from_le_bytes(head.crc); - let mut iv = [0u8; 16]; - let mut expected_tag = [0u8; 16]; - reader.read_exact(&mut iv)?; - reader.read_exact(&mut expected_tag)?; - let csum_reader = ChecksumReader::new(reader, None); - let decrypt_reader = CryptReader::new(BufReader::with_capacity(64*1024, csum_reader), iv, expected_tag, config.unwrap())?; - Ok(Self { state: BlobReaderState::Encrypted { expected_crc, decrypt_reader }}) - } - ENCR_COMPR_BLOB_MAGIC_1_0 => { - let expected_crc = u32::from_le_bytes(head.crc); - let mut iv = [0u8; 16]; - let mut expected_tag = [0u8; 16]; - reader.read_exact(&mut iv)?; - reader.read_exact(&mut expected_tag)?; - let csum_reader = ChecksumReader::new(reader, None); - let decrypt_reader = CryptReader::new(BufReader::with_capacity(64*1024, csum_reader), iv, expected_tag, config.unwrap())?; - let decompr = zstd::stream::read::Decoder::new(decrypt_reader)?; - Ok(Self { state: BlobReaderState::EncryptedCompressed { expected_crc, decompr }}) - } - _ => bail!("got wrong magic number {:?}", head.magic) - } - } - - pub fn finish(self) -> Result { - match self.state { - BlobReaderState::Uncompressed { csum_reader, expected_crc } => { - let (reader, crc, _) = csum_reader.finish()?; - if crc != expected_crc { - bail!("blob crc check failed"); - } - Ok(reader) - } - BlobReaderState::Compressed { expected_crc, decompr } => { - let csum_reader = decompr.finish().into_inner(); - let (reader, crc, _) = csum_reader.finish()?; - if crc != expected_crc { - bail!("blob crc check failed"); - } - Ok(reader) - } - BlobReaderState::Signed { csum_reader, expected_crc, expected_hmac } => { - let (reader, crc, hmac) = csum_reader.finish()?; - if crc != expected_crc { - bail!("blob crc check failed"); - } - if let Some(hmac) = hmac { - if hmac != expected_hmac { - bail!("blob signature check failed"); - } - } - Ok(reader) - } - BlobReaderState::SignedCompressed { expected_crc, expected_hmac, decompr } => { - let csum_reader = decompr.finish().into_inner(); - let (reader, crc, hmac) = csum_reader.finish()?; - if crc != expected_crc { - bail!("blob crc check failed"); - } - if let Some(hmac) = hmac { - if hmac != expected_hmac { - bail!("blob signature check failed"); - } - } - Ok(reader) - } - BlobReaderState::Encrypted { expected_crc, decrypt_reader } => { - let csum_reader = decrypt_reader.finish()?.into_inner(); - let (reader, crc, _) = csum_reader.finish()?; - if crc != expected_crc { - bail!("blob crc check failed"); - } - Ok(reader) - } - BlobReaderState::EncryptedCompressed { expected_crc, decompr } => { - let decrypt_reader = decompr.finish().into_inner(); - let csum_reader = decrypt_reader.finish()?.into_inner(); - let (reader, crc, _) = csum_reader.finish()?; - if crc != expected_crc { - bail!("blob crc check failed"); - } - Ok(reader) - } - } - } -} - -impl <'a, R: BufRead> Read for DataBlobReader<'a, R> { - - fn read(&mut self, buf: &mut [u8]) -> Result { - match &mut self.state { - BlobReaderState::Uncompressed { csum_reader, .. } => { - csum_reader.read(buf) - } - BlobReaderState::Compressed { decompr, .. } => { - decompr.read(buf) - } - BlobReaderState::Signed { csum_reader, .. } => { - csum_reader.read(buf) - } - BlobReaderState::SignedCompressed { decompr, .. } => { - decompr.read(buf) - } - BlobReaderState::Encrypted { decrypt_reader, .. } => { - decrypt_reader.read(buf) - } - BlobReaderState::EncryptedCompressed { decompr, .. } => { - decompr.read(buf) - } - } - } } diff --git a/src/backup/data_blob_reader.rs b/src/backup/data_blob_reader.rs new file mode 100644 index 00000000..e1572eb2 --- /dev/null +++ b/src/backup/data_blob_reader.rs @@ -0,0 +1,169 @@ +use failure::*; +use std::io::{Read, BufRead, BufReader}; +use proxmox::tools::io::ReadExt; + +use super::*; + +enum BlobReaderState<'a, R: Read> { + Uncompressed { expected_crc: u32, csum_reader: ChecksumReader<'a, R> }, + Compressed { expected_crc: u32, decompr: zstd::stream::read::Decoder>> }, + Signed { expected_crc: u32, expected_hmac: [u8; 32], csum_reader: ChecksumReader<'a, R> }, + SignedCompressed { expected_crc: u32, expected_hmac: [u8; 32], decompr: zstd::stream::read::Decoder>> }, + Encrypted { expected_crc: u32, decrypt_reader: CryptReader>> }, + EncryptedCompressed { expected_crc: u32, decompr: zstd::stream::read::Decoder>>>> }, +} + +/// Read data blobs +pub struct DataBlobReader<'a, R: Read> { + state: BlobReaderState<'a, R>, +} + +impl <'a, R: Read> DataBlobReader<'a, R> { + + pub fn new(mut reader: R, config: Option<&'a CryptConfig>) -> Result { + + let head: DataBlobHeader = unsafe { reader.read_le_value()? }; + match head.magic { + UNCOMPRESSED_BLOB_MAGIC_1_0 => { + let expected_crc = u32::from_le_bytes(head.crc); + let csum_reader = ChecksumReader::new(reader, None); + Ok(Self { state: BlobReaderState::Uncompressed { expected_crc, csum_reader }}) + } + COMPRESSED_BLOB_MAGIC_1_0 => { + let expected_crc = u32::from_le_bytes(head.crc); + let csum_reader = ChecksumReader::new(reader, None); + + let decompr = zstd::stream::read::Decoder::new(csum_reader)?; + Ok(Self { state: BlobReaderState::Compressed { expected_crc, decompr }}) + } + AUTHENTICATED_BLOB_MAGIC_1_0 => { + let expected_crc = u32::from_le_bytes(head.crc); + let mut expected_hmac = [0u8; 32]; + reader.read_exact(&mut expected_hmac)?; + let signer = config.map(|c| c.data_signer()); + let csum_reader = ChecksumReader::new(reader, signer); + Ok(Self { state: BlobReaderState::Signed { expected_crc, expected_hmac, csum_reader }}) + } + AUTH_COMPR_BLOB_MAGIC_1_0 => { + let expected_crc = u32::from_le_bytes(head.crc); + let mut expected_hmac = [0u8; 32]; + reader.read_exact(&mut expected_hmac)?; + let signer = config.map(|c| c.data_signer()); + let csum_reader = ChecksumReader::new(reader, signer); + + let decompr = zstd::stream::read::Decoder::new(csum_reader)?; + Ok(Self { state: BlobReaderState::SignedCompressed { expected_crc, expected_hmac, decompr }}) + } + ENCRYPTED_BLOB_MAGIC_1_0 => { + let expected_crc = u32::from_le_bytes(head.crc); + let mut iv = [0u8; 16]; + let mut expected_tag = [0u8; 16]; + reader.read_exact(&mut iv)?; + reader.read_exact(&mut expected_tag)?; + let csum_reader = ChecksumReader::new(reader, None); + let decrypt_reader = CryptReader::new(BufReader::with_capacity(64*1024, csum_reader), iv, expected_tag, config.unwrap())?; + Ok(Self { state: BlobReaderState::Encrypted { expected_crc, decrypt_reader }}) + } + ENCR_COMPR_BLOB_MAGIC_1_0 => { + let expected_crc = u32::from_le_bytes(head.crc); + let mut iv = [0u8; 16]; + let mut expected_tag = [0u8; 16]; + reader.read_exact(&mut iv)?; + reader.read_exact(&mut expected_tag)?; + let csum_reader = ChecksumReader::new(reader, None); + let decrypt_reader = CryptReader::new(BufReader::with_capacity(64*1024, csum_reader), iv, expected_tag, config.unwrap())?; + let decompr = zstd::stream::read::Decoder::new(decrypt_reader)?; + Ok(Self { state: BlobReaderState::EncryptedCompressed { expected_crc, decompr }}) + } + _ => bail!("got wrong magic number {:?}", head.magic) + } + } + + pub fn finish(self) -> Result { + match self.state { + BlobReaderState::Uncompressed { csum_reader, expected_crc } => { + let (reader, crc, _) = csum_reader.finish()?; + if crc != expected_crc { + bail!("blob crc check failed"); + } + Ok(reader) + } + BlobReaderState::Compressed { expected_crc, decompr } => { + let csum_reader = decompr.finish().into_inner(); + let (reader, crc, _) = csum_reader.finish()?; + if crc != expected_crc { + bail!("blob crc check failed"); + } + Ok(reader) + } + BlobReaderState::Signed { csum_reader, expected_crc, expected_hmac } => { + let (reader, crc, hmac) = csum_reader.finish()?; + if crc != expected_crc { + bail!("blob crc check failed"); + } + if let Some(hmac) = hmac { + if hmac != expected_hmac { + bail!("blob signature check failed"); + } + } + Ok(reader) + } + BlobReaderState::SignedCompressed { expected_crc, expected_hmac, decompr } => { + let csum_reader = decompr.finish().into_inner(); + let (reader, crc, hmac) = csum_reader.finish()?; + if crc != expected_crc { + bail!("blob crc check failed"); + } + if let Some(hmac) = hmac { + if hmac != expected_hmac { + bail!("blob signature check failed"); + } + } + Ok(reader) + } + BlobReaderState::Encrypted { expected_crc, decrypt_reader } => { + let csum_reader = decrypt_reader.finish()?.into_inner(); + let (reader, crc, _) = csum_reader.finish()?; + if crc != expected_crc { + bail!("blob crc check failed"); + } + Ok(reader) + } + BlobReaderState::EncryptedCompressed { expected_crc, decompr } => { + let decrypt_reader = decompr.finish().into_inner(); + let csum_reader = decrypt_reader.finish()?.into_inner(); + let (reader, crc, _) = csum_reader.finish()?; + if crc != expected_crc { + bail!("blob crc check failed"); + } + Ok(reader) + } + } + } +} + +impl <'a, R: BufRead> Read for DataBlobReader<'a, R> { + + fn read(&mut self, buf: &mut [u8]) -> Result { + match &mut self.state { + BlobReaderState::Uncompressed { csum_reader, .. } => { + csum_reader.read(buf) + } + BlobReaderState::Compressed { decompr, .. } => { + decompr.read(buf) + } + BlobReaderState::Signed { csum_reader, .. } => { + csum_reader.read(buf) + } + BlobReaderState::SignedCompressed { decompr, .. } => { + decompr.read(buf) + } + BlobReaderState::Encrypted { decrypt_reader, .. } => { + decrypt_reader.read(buf) + } + BlobReaderState::EncryptedCompressed { decompr, .. } => { + decompr.read(buf) + } + } + } +} diff --git a/src/backup/data_blob_writer.rs b/src/backup/data_blob_writer.rs new file mode 100644 index 00000000..cf024dc5 --- /dev/null +++ b/src/backup/data_blob_writer.rs @@ -0,0 +1,244 @@ +use failure::*; +use std::io::{Write, Seek, SeekFrom}; +use proxmox::tools::io::WriteExt; + +use super::*; + +enum BlobWriterState<'a, W: Write> { + Uncompressed { csum_writer: ChecksumWriter<'a, W> }, + Compressed { compr: zstd::stream::write::Encoder> }, + Signed { csum_writer: ChecksumWriter<'a, W> }, + SignedCompressed { compr: zstd::stream::write::Encoder> }, + Encrypted { crypt_writer: CryptWriter> }, + EncryptedCompressed { compr: zstd::stream::write::Encoder>> }, +} + +/// Data blob writer +pub struct DataBlobWriter<'a, W: Write> { + state: BlobWriterState<'a, W>, +} + +impl <'a, W: Write + Seek> DataBlobWriter<'a, W> { + + pub fn new_uncompressed(mut writer: W) -> Result { + writer.seek(SeekFrom::Start(0))?; + let head = DataBlobHeader { magic: UNCOMPRESSED_BLOB_MAGIC_1_0, crc: [0; 4] }; + unsafe { + writer.write_le_value(head)?; + } + let csum_writer = ChecksumWriter::new(writer, None); + Ok(Self { state: BlobWriterState::Uncompressed { csum_writer }}) + } + + pub fn new_compressed(mut writer: W) -> Result { + writer.seek(SeekFrom::Start(0))?; + let head = DataBlobHeader { magic: COMPRESSED_BLOB_MAGIC_1_0, crc: [0; 4] }; + unsafe { + writer.write_le_value(head)?; + } + let csum_writer = ChecksumWriter::new(writer, None); + let compr = zstd::stream::write::Encoder::new(csum_writer, 1)?; + Ok(Self { state: BlobWriterState::Compressed { compr }}) + } + + pub fn new_signed(mut writer: W, config: &'a CryptConfig) -> Result { + writer.seek(SeekFrom::Start(0))?; + let head = AuthenticatedDataBlobHeader { + head: DataBlobHeader { magic: AUTHENTICATED_BLOB_MAGIC_1_0, crc: [0; 4] }, + tag: [0u8; 32], + }; + unsafe { + writer.write_le_value(head)?; + } + let signer = config.data_signer(); + let csum_writer = ChecksumWriter::new(writer, Some(signer)); + Ok(Self { state: BlobWriterState::Signed { csum_writer }}) + } + + pub fn new_signed_compressed(mut writer: W, config: &'a CryptConfig) -> Result { + writer.seek(SeekFrom::Start(0))?; + let head = AuthenticatedDataBlobHeader { + head: DataBlobHeader { magic: AUTH_COMPR_BLOB_MAGIC_1_0, crc: [0; 4] }, + tag: [0u8; 32], + }; + unsafe { + writer.write_le_value(head)?; + } + let signer = config.data_signer(); + let csum_writer = ChecksumWriter::new(writer, Some(signer)); + let compr = zstd::stream::write::Encoder::new(csum_writer, 1)?; + Ok(Self { state: BlobWriterState::SignedCompressed { compr }}) + } + + pub fn new_encrypted(mut writer: W, config: &'a CryptConfig) -> Result { + writer.seek(SeekFrom::Start(0))?; + let head = EncryptedDataBlobHeader { + head: DataBlobHeader { magic: ENCRYPTED_BLOB_MAGIC_1_0, crc: [0; 4] }, + iv: [0u8; 16], + tag: [0u8; 16], + }; + unsafe { + writer.write_le_value(head)?; + } + + let csum_writer = ChecksumWriter::new(writer, None); + let crypt_writer = CryptWriter::new(csum_writer, config)?; + Ok(Self { state: BlobWriterState::Encrypted { crypt_writer }}) + } + + pub fn new_encrypted_compressed(mut writer: W, config: &'a CryptConfig) -> Result { + writer.seek(SeekFrom::Start(0))?; + let head = EncryptedDataBlobHeader { + head: DataBlobHeader { magic: ENCR_COMPR_BLOB_MAGIC_1_0, crc: [0; 4] }, + iv: [0u8; 16], + tag: [0u8; 16], + }; + unsafe { + writer.write_le_value(head)?; + } + + let csum_writer = ChecksumWriter::new(writer, None); + let crypt_writer = CryptWriter::new(csum_writer, config)?; + let compr = zstd::stream::write::Encoder::new(crypt_writer, 1)?; + Ok(Self { state: BlobWriterState::EncryptedCompressed { compr }}) + } + + pub fn finish(self) -> Result { + match self.state { + BlobWriterState::Uncompressed { csum_writer } => { + // write CRC + let (mut writer, crc, _) = csum_writer.finish()?; + let head = DataBlobHeader { magic: UNCOMPRESSED_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }; + + writer.seek(SeekFrom::Start(0))?; + unsafe { + writer.write_le_value(head)?; + } + + return Ok(writer) + } + BlobWriterState::Compressed { compr } => { + let csum_writer = compr.finish()?; + let (mut writer, crc, _) = csum_writer.finish()?; + + let head = DataBlobHeader { magic: COMPRESSED_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }; + + writer.seek(SeekFrom::Start(0))?; + unsafe { + writer.write_le_value(head)?; + } + + return Ok(writer) + } + BlobWriterState::Signed { csum_writer } => { + let (mut writer, crc, tag) = csum_writer.finish()?; + + let head = AuthenticatedDataBlobHeader { + head: DataBlobHeader { magic: AUTHENTICATED_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }, + tag: tag.unwrap(), + }; + + writer.seek(SeekFrom::Start(0))?; + unsafe { + writer.write_le_value(head)?; + } + + return Ok(writer) + } + BlobWriterState::SignedCompressed { compr } => { + let csum_writer = compr.finish()?; + let (mut writer, crc, tag) = csum_writer.finish()?; + + let head = AuthenticatedDataBlobHeader { + head: DataBlobHeader { magic: AUTH_COMPR_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }, + tag: tag.unwrap(), + }; + + writer.seek(SeekFrom::Start(0))?; + unsafe { + writer.write_le_value(head)?; + } + + return Ok(writer) + } + BlobWriterState::Encrypted { crypt_writer } => { + let (csum_writer, iv, tag) = crypt_writer.finish()?; + let (mut writer, crc, _) = csum_writer.finish()?; + + let head = EncryptedDataBlobHeader { + head: DataBlobHeader { magic: ENCRYPTED_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }, + iv, tag, + }; + writer.seek(SeekFrom::Start(0))?; + unsafe { + writer.write_le_value(head)?; + } + return Ok(writer) + } + BlobWriterState::EncryptedCompressed { compr } => { + let crypt_writer = compr.finish()?; + let (csum_writer, iv, tag) = crypt_writer.finish()?; + let (mut writer, crc, _) = csum_writer.finish()?; + + let head = EncryptedDataBlobHeader { + head: DataBlobHeader { magic: ENCR_COMPR_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() }, + iv, tag, + }; + writer.seek(SeekFrom::Start(0))?; + unsafe { + writer.write_le_value(head)?; + } + return Ok(writer) + } + } + } +} + +impl <'a, W: Write + Seek> Write for DataBlobWriter<'a, W> { + + fn write(&mut self, buf: &[u8]) -> Result { + match self.state { + BlobWriterState::Uncompressed { ref mut csum_writer } => { + csum_writer.write(buf) + } + BlobWriterState::Compressed { ref mut compr } => { + compr.write(buf) + } + BlobWriterState::Signed { ref mut csum_writer } => { + csum_writer.write(buf) + } + BlobWriterState::SignedCompressed { ref mut compr } => { + compr.write(buf) + } + BlobWriterState::Encrypted { ref mut crypt_writer } => { + crypt_writer.write(buf) + } + BlobWriterState::EncryptedCompressed { ref mut compr } => { + compr.write(buf) + } + } + } + + fn flush(&mut self) -> Result<(), std::io::Error> { + match self.state { + BlobWriterState::Uncompressed { ref mut csum_writer } => { + csum_writer.flush() + } + BlobWriterState::Compressed { ref mut compr } => { + compr.flush() + } + BlobWriterState::Signed { ref mut csum_writer } => { + csum_writer.flush() + } + BlobWriterState::SignedCompressed { ref mut compr } => { + compr.flush() + } + BlobWriterState::Encrypted { ref mut crypt_writer } => { + crypt_writer.flush() + } + BlobWriterState::EncryptedCompressed { ref mut compr } => { + compr.flush() + } + } + } +}