tyler
/
proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

avoid lifetimes in blob reader/writer

This commit is contained in:
Wolfgang Bumiller 2019-08-16 09:19:01 +02:00 committed by Dietmar Maurer
parent 71d08e00b7
commit 9025312aa6
10 changed files with 98 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/backup/checksum_reader.rs
View File

@ -1,16 +1,30 @@
use failure::*;
use std::sync::Arc;
use std::io::Read;
pub struct ChecksumReader<'a, R> {
use super::CryptConfig;
use crate::tools::borrow::Tied;
pub struct ChecksumReader<R> {
reader: R,
hasher: crc32fast::Hasher,
signer: Option<openssl::sign::Signer<'a>>,
signer: Option<Tied<Arc<CryptConfig>, openssl::sign::Signer<'static>>>,
}
impl <'a, R: Read> ChecksumReader<'a, R> {
impl <R: Read> ChecksumReader<R> {
pub fn new(reader: R, signer: Option<openssl::sign::Signer<'a>>) -> Self {
pub fn new(reader: R, config: Option<Arc<CryptConfig>>) -> Self {
let hasher = crc32fast::Hasher::new();
let signer = match config {
Some(config) => {
let tied_signer = Tied::new(config, |config| {
Box::new(unsafe { (*config).data_signer() })
});
Some(tied_signer)
}
None => None,
};
Self { reader, hasher, signer }
}
@ -27,7 +41,7 @@ impl <'a, R: Read> ChecksumReader<'a, R> {
}
}
impl <'a, R: Read> Read for ChecksumReader<'a, R> {
impl <R: Read> Read for ChecksumReader<R> {
fn read(&mut self, buf: &mut [u8]) -> Result<usize, std::io::Error> {
let count = self.reader.read(buf)?;

26
src/backup/checksum_writer.rs
View File

@ -1,16 +1,30 @@
use failure::*;
use std::sync::Arc;
use std::io::Write;
pub struct ChecksumWriter<'a, W> {
use failure::*;
use super::CryptConfig;
use crate::tools::borrow::Tied;
pub struct ChecksumWriter<W> {
writer: W,
hasher: crc32fast::Hasher,
signer: Option<openssl::sign::Signer<'a>>,
signer: Option<Tied<Arc<CryptConfig>, openssl::sign::Signer<'static>>>,
}
impl <'a, W: Write> ChecksumWriter<'a, W> {
impl <W: Write> ChecksumWriter<W> {
pub fn new(writer: W, signer: Option<openssl::sign::Signer<'a>>) -> Self {
pub fn new(writer: W, config: Option<Arc<CryptConfig>>) -> Self {
let hasher = crc32fast::Hasher::new();
let signer = match config {
Some(config) => {
let tied_signer = Tied::new(config.clone(), |config| {
Box::new(unsafe { (*config).data_signer() })
});
Some(tied_signer)
}
None => None,
};
Self { writer, hasher, signer }
}
@ -27,7 +41,7 @@ impl <'a, W: Write> ChecksumWriter<'a, W> {
}
}
impl <'a, W: Write> Write for ChecksumWriter<'a, W> {
impl <W: Write> Write for ChecksumWriter<W> {
fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
self.hasher.update(buf);

3
src/backup/crypt_reader.rs
View File

@ -1,4 +1,5 @@
use failure::*;
use std::sync::Arc;
use std::io::{Read, BufRead};
use super::CryptConfig;
@ -13,7 +14,7 @@ pub struct CryptReader<R> {
impl <R: BufRead> CryptReader<R> {
pub fn new(reader: R, iv: [u8; 16], tag: [u8; 16], config: &CryptConfig) -> Result<Self, Error> {
pub fn new(reader: R, iv: [u8; 16], tag: [u8; 16], config: Arc<CryptConfig>) -> Result<Self, Error> {
let block_size = config.cipher().block_size(); // Note: block size is normally 1 byte for stream ciphers
if block_size.count_ones() != 1 || block_size > 512 {
bail!("unexpected Cipher block size {}", block_size);

3
src/backup/crypt_writer.rs
View File

@ -1,4 +1,5 @@
use failure::*;
use std::sync::Arc;
use std::io::Write;
use super::CryptConfig;
@ -13,7 +14,7 @@ pub struct CryptWriter<W> {
impl <W: Write> CryptWriter<W> {
pub fn new(writer: W, config: &CryptConfig) -> Result<Self, Error> {
pub fn new(writer: W, config: Arc<CryptConfig>) -> Result<Self, Error> {
let mut iv = [0u8; 16];
proxmox::sys::linux::fill_with_random_data(&mut iv)?;
let block_size = config.cipher().block_size();

7
src/backup/data_blob.rs
View File

@ -1,5 +1,6 @@
use failure::*;
use std::convert::TryInto;
use std::sync::Arc;
use proxmox::tools::io::{ReadExt, WriteExt};
@ -69,7 +70,7 @@ impl DataBlob {
/// Create a DataBlob, optionally compressed and/or encrypted
pub fn encode(
data: &[u8],
config: Option<&CryptConfig>,
config: Option<Arc<CryptConfig>>,
compress: bool,
) -> Result<Self, Error> {
@ -158,7 +159,7 @@ impl DataBlob {
}
/// Decode blob data
pub fn decode(self, config: Option<&CryptConfig>) -> Result<Vec<u8>, Error> {
pub fn decode(self, config: Option<Arc<CryptConfig>>) -> Result<Vec<u8>, Error> {
let magic = self.magic();
@ -215,7 +216,7 @@ impl DataBlob {
/// Create a signed DataBlob, optionally compressed
pub fn create_signed(
data: &[u8],
config: &CryptConfig,
config: Arc<CryptConfig>,
compress: bool,
) -> Result<Self, Error> {

31
src/backup/data_blob_reader.rs
View File

@ -1,26 +1,27 @@
use failure::*;
use std::sync::Arc;
use std::io::{Read, BufReader};
use proxmox::tools::io::ReadExt;
use super::*;
enum BlobReaderState<'a, R: Read> {
Uncompressed { expected_crc: u32, csum_reader: ChecksumReader<'a, R> },
Compressed { expected_crc: u32, decompr: zstd::stream::read::Decoder<BufReader<ChecksumReader<'a, R>>> },
Signed { expected_crc: u32, expected_hmac: [u8; 32], csum_reader: ChecksumReader<'a, R> },
SignedCompressed { expected_crc: u32, expected_hmac: [u8; 32], decompr: zstd::stream::read::Decoder<BufReader<ChecksumReader<'a, R>>> },
Encrypted { expected_crc: u32, decrypt_reader: CryptReader<BufReader<ChecksumReader<'a, R>>> },
EncryptedCompressed { expected_crc: u32, decompr: zstd::stream::read::Decoder<BufReader<CryptReader<BufReader<ChecksumReader<'a, R>>>>> },
enum BlobReaderState<R: Read> {
Uncompressed { expected_crc: u32, csum_reader: ChecksumReader<R> },
Compressed { expected_crc: u32, decompr: zstd::stream::read::Decoder<BufReader<ChecksumReader<R>>> },
Signed { expected_crc: u32, expected_hmac: [u8; 32], csum_reader: ChecksumReader<R> },
SignedCompressed { expected_crc: u32, expected_hmac: [u8; 32], decompr: zstd::stream::read::Decoder<BufReader<ChecksumReader<R>>> },
Encrypted { expected_crc: u32, decrypt_reader: CryptReader<BufReader<ChecksumReader<R>>> },
EncryptedCompressed { expected_crc: u32, decompr: zstd::stream::read::Decoder<BufReader<CryptReader<BufReader<ChecksumReader<R>>>>> },
}
/// Read data blobs
pub struct DataBlobReader<'a, R: Read> {
state: BlobReaderState<'a, R>,
pub struct DataBlobReader<R: Read> {
state: BlobReaderState<R>,
}
impl <'a, R: Read> DataBlobReader<'a, R> {
impl <R: Read> DataBlobReader<R> {
pub fn new(mut reader: R, config: Option<&'a CryptConfig>) -> Result<Self, Error> {
pub fn new(mut reader: R, config: Option<Arc<CryptConfig>>) -> Result<Self, Error> {
let head: DataBlobHeader = unsafe { reader.read_le_value()? };
match head.magic {
@ -40,16 +41,14 @@ impl <'a, R: Read> DataBlobReader<'a, R> {
let expected_crc = u32::from_le_bytes(head.crc);
let mut expected_hmac = [0u8; 32];
reader.read_exact(&mut expected_hmac)?;
let signer = config.map(|c| c.data_signer());
let csum_reader = ChecksumReader::new(reader, signer);
let csum_reader = ChecksumReader::new(reader, config);
Ok(Self { state: BlobReaderState::Signed { expected_crc, expected_hmac, csum_reader }})
}
AUTH_COMPR_BLOB_MAGIC_1_0 => {
let expected_crc = u32::from_le_bytes(head.crc);
let mut expected_hmac = [0u8; 32];
reader.read_exact(&mut expected_hmac)?;
let signer = config.map(|c| c.data_signer());
let csum_reader = ChecksumReader::new(reader, signer);
let csum_reader = ChecksumReader::new(reader, config);
let decompr = zstd::stream::read::Decoder::new(csum_reader)?;
Ok(Self { state: BlobReaderState::SignedCompressed { expected_crc, expected_hmac, decompr }})
@ -142,7 +141,7 @@ impl <'a, R: Read> DataBlobReader<'a, R> {
}
}
impl <'a, R: Read> Read for DataBlobReader<'a, R> {
impl <R: Read> Read for DataBlobReader<R> {
fn read(&mut self, buf: &mut [u8]) -> Result<usize, std::io::Error> {
match &mut self.state {

37
src/backup/data_blob_writer.rs
View File

@ -1,24 +1,25 @@
use failure::*;
use std::sync::Arc;
use std::io::{Write, Seek, SeekFrom};
use proxmox::tools::io::WriteExt;
use super::*;
enum BlobWriterState<'a, W: Write> {
Uncompressed { csum_writer: ChecksumWriter<'a, W> },
Compressed { compr: zstd::stream::write::Encoder<ChecksumWriter<'a, W>> },
Signed { csum_writer: ChecksumWriter<'a, W> },
SignedCompressed { compr: zstd::stream::write::Encoder<ChecksumWriter<'a, W>> },
Encrypted { crypt_writer: CryptWriter<ChecksumWriter<'a, W>> },
EncryptedCompressed { compr: zstd::stream::write::Encoder<CryptWriter<ChecksumWriter<'a, W>>> },
enum BlobWriterState<W: Write> {
Uncompressed { csum_writer: ChecksumWriter<W> },
Compressed { compr: zstd::stream::write::Encoder<ChecksumWriter<W>> },
Signed { csum_writer: ChecksumWriter<W> },
SignedCompressed { compr: zstd::stream::write::Encoder<ChecksumWriter<W>> },
Encrypted { crypt_writer: CryptWriter<ChecksumWriter<W>> },
EncryptedCompressed { compr: zstd::stream::write::Encoder<CryptWriter<ChecksumWriter<W>>> },
}
/// Data blob writer
pub struct DataBlobWriter<'a, W: Write> {
state: BlobWriterState<'a, W>,
pub struct DataBlobWriter<W: Write> {
state: BlobWriterState<W>,
}
impl <'a, W: Write + Seek> DataBlobWriter<'a, W> {
impl <W: Write + Seek> DataBlobWriter<W> {
pub fn new_uncompressed(mut writer: W) -> Result<Self, Error> {
writer.seek(SeekFrom::Start(0))?;
@ -41,7 +42,7 @@ impl <'a, W: Write + Seek> DataBlobWriter<'a, W> {
Ok(Self { state: BlobWriterState::Compressed { compr }})
}
pub fn new_signed(mut writer: W, config: &'a CryptConfig) -> Result<Self, Error> {
pub fn new_signed(mut writer: W, config: Arc<CryptConfig>) -> Result<Self, Error> {
writer.seek(SeekFrom::Start(0))?;
let head = AuthenticatedDataBlobHeader {
head: DataBlobHeader { magic: AUTHENTICATED_BLOB_MAGIC_1_0, crc: [0; 4] },
@ -50,12 +51,11 @@ impl <'a, W: Write + Seek> DataBlobWriter<'a, W> {
unsafe {
writer.write_le_value(head)?;
}
let signer = config.data_signer();
let csum_writer = ChecksumWriter::new(writer, Some(signer));
let csum_writer = ChecksumWriter::new(writer, Some(config));
Ok(Self { state: BlobWriterState::Signed { csum_writer }})
}
pub fn new_signed_compressed(mut writer: W, config: &'a CryptConfig) -> Result<Self, Error> {
pub fn new_signed_compressed(mut writer: W, config: Arc<CryptConfig>) -> Result<Self, Error> {
writer.seek(SeekFrom::Start(0))?;
let head = AuthenticatedDataBlobHeader {
head: DataBlobHeader { magic: AUTH_COMPR_BLOB_MAGIC_1_0, crc: [0; 4] },
@ -64,13 +64,12 @@ impl <'a, W: Write + Seek> DataBlobWriter<'a, W> {
unsafe {
writer.write_le_value(head)?;
}
let signer = config.data_signer();
let csum_writer = ChecksumWriter::new(writer, Some(signer));
let csum_writer = ChecksumWriter::new(writer, Some(config));
let compr = zstd::stream::write::Encoder::new(csum_writer, 1)?;
Ok(Self { state: BlobWriterState::SignedCompressed { compr }})
}
pub fn new_encrypted(mut writer: W, config: &'a CryptConfig) -> Result<Self, Error> {
pub fn new_encrypted(mut writer: W, config: Arc<CryptConfig>) -> Result<Self, Error> {
writer.seek(SeekFrom::Start(0))?;
let head = EncryptedDataBlobHeader {
head: DataBlobHeader { magic: ENCRYPTED_BLOB_MAGIC_1_0, crc: [0; 4] },
@ -86,7 +85,7 @@ impl <'a, W: Write + Seek> DataBlobWriter<'a, W> {
Ok(Self { state: BlobWriterState::Encrypted { crypt_writer }})
}
pub fn new_encrypted_compressed(mut writer: W, config: &'a CryptConfig) -> Result<Self, Error> {
pub fn new_encrypted_compressed(mut writer: W, config: Arc<CryptConfig>) -> Result<Self, Error> {
writer.seek(SeekFrom::Start(0))?;
let head = EncryptedDataBlobHeader {
head: DataBlobHeader { magic: ENCR_COMPR_BLOB_MAGIC_1_0, crc: [0; 4] },
@ -194,7 +193,7 @@ impl <'a, W: Write + Seek> DataBlobWriter<'a, W> {
}
}
impl <'a, W: Write + Seek> Write for DataBlobWriter<'a, W> {
impl <W: Write + Seek> Write for DataBlobWriter<W> {
fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
match self.state {

17
src/bin/proxmox-backup-client.rs
View File

@ -435,7 +435,7 @@ fn dump_catalog(
None => None,
Some(path) => {
let (key, _) = load_and_decrtypt_key(&path, get_encryption_key_password)?;
Some(CryptConfig::new(key)?)
Some(Arc::new(CryptConfig::new(key)?))
}
};
@ -457,7 +457,7 @@ fn dump_catalog(
blob_file.seek(SeekFrom::Start(0))?;
let reader = BufReader::new(DataBlobReader::new(blob_file, crypt_config.as_ref())?);
let reader = BufReader::new(DataBlobReader::new(blob_file, crypt_config)?);
let mut catalog_reader = pxar::catalog::SimpleCatalogReader::new(reader);
@ -888,10 +888,7 @@ fn restore(
let blob = DataBlob::from_raw(blob_data)?;
blob.verify_crc()?;
let raw_data = match crypt_config {
Some(ref crypt_config) => blob.decode(Some(crypt_config))?,
None => blob.decode(None)?,
};
let raw_data = blob.decode(crypt_config)?;
if let Some(target) = target {
file_set_contents(target, &raw_data, None)?;
@ -991,17 +988,13 @@ fn upload_log(
Some(path) => {
let (key, _created) = load_and_decrtypt_key(&path, get_encryption_key_password)?;
let crypt_config = CryptConfig::new(key)?;
Some(crypt_config)
Some(Arc::new(crypt_config))
}
};
let data = file_get_contents(logfile)?;
let blob = if let Some(ref crypt_config) = crypt_config {
DataBlob::encode(&data, Some(crypt_config), true)?
} else {
DataBlob::encode(&data, None, true)?
};
let blob = DataBlob::encode(&data, crypt_config, true)?;
let raw_data = blob.into_inner();

10
src/client/http_client.rs
View File

@ -641,11 +641,11 @@ impl BackupClient {
futures::future::ok(())
.and_then(move |_| {
let blob = if let Some(ref crypt_config) = crypt_config {
let blob = if let Some(crypt_config) = crypt_config {
if sign_only {
DataBlob::create_signed(&data, crypt_config, compress)?
} else {
DataBlob::encode(&data, Some(crypt_config), compress)?
DataBlob::encode(&data, Some(crypt_config.clone()), compress)?
}
} else {
DataBlob::encode(&data, None, compress)?
@ -683,11 +683,7 @@ impl BackupClient {
tokio::io::read_to_end(file, contents)
.map_err(Error::from)
.and_then(move |(_, contents)| {
let blob = if let Some(ref crypt_config) = crypt_config {
DataBlob::encode(&contents, Some(crypt_config), compress)?
} else {
DataBlob::encode(&contents, None, compress)?
};
let blob = DataBlob::encode(&contents, crypt_config, compress)?;
let raw_data = blob.into_inner();
Ok((raw_data, contents.len() as u64))
})

19
tests/blob_writer.rs
View File

@ -1,6 +1,7 @@
use failure::*;
use std::sync::Arc;
use std::io::Cursor;
use std::io::{ Read, Write, Seek, SeekFrom };
use std::io::{Read, Write, Seek, SeekFrom };
use lazy_static::lazy_static;
use proxmox_backup::backup::*;
@ -16,9 +17,9 @@ lazy_static! {
data
};
static ref CRYPT_CONFIG: CryptConfig = {
static ref CRYPT_CONFIG: Arc<CryptConfig> = {
let key = [1u8; 32];
CryptConfig::new(key).unwrap()
Arc::new(CryptConfig::new(key).unwrap())
};
}
@ -30,7 +31,7 @@ fn verify_test_blob(mut cursor: Cursor<Vec<u8>>) -> Result<(), Error> {
println!("Starting DataBlobReader test (size = {})", size);
cursor.seek(SeekFrom::Start(0))?;
let mut reader = DataBlobReader::new(&mut cursor, Some(&CRYPT_CONFIG))?;
let mut reader = DataBlobReader::new(&mut cursor, Some(CRYPT_CONFIG.clone()))?;
let mut buffer = Vec::<u8>::new();
// read the whole file
//reader.read_to_end(&mut buffer)?;
@ -52,7 +53,7 @@ fn verify_test_blob(mut cursor: Cursor<Vec<u8>>) -> Result<(), Error> {
let blob = DataBlob::from_raw(raw_data)?;
blob.verify_crc()?;
let data = blob.decode(Some(&CRYPT_CONFIG))?;
let data = blob.decode(Some(CRYPT_CONFIG.clone()))?;
if data != *TEST_DATA {
bail!("blob data is wrong (decode)");
}
@ -80,7 +81,7 @@ fn test_compressed_blob_writer() -> Result<(), Error> {
#[test]
fn test_signed_blob_writer() -> Result<(), Error> {
let tmp = Cursor::new(Vec::<u8>::new());
let mut blob_writer = DataBlobWriter::new_signed(tmp, &CRYPT_CONFIG)?;
let mut blob_writer = DataBlobWriter::new_signed(tmp, CRYPT_CONFIG.clone())?;
blob_writer.write_all(&TEST_DATA)?;
verify_test_blob(blob_writer.finish()?)
@ -89,7 +90,7 @@ fn test_signed_blob_writer() -> Result<(), Error> {
#[test]
fn test_signed_compressed_blob_writer() -> Result<(), Error> {
let tmp = Cursor::new(Vec::<u8>::new());
let mut blob_writer = DataBlobWriter::new_signed_compressed(tmp, &CRYPT_CONFIG)?;
let mut blob_writer = DataBlobWriter::new_signed_compressed(tmp, CRYPT_CONFIG.clone())?;
blob_writer.write_all(&TEST_DATA)?;
verify_test_blob(blob_writer.finish()?)
@ -98,7 +99,7 @@ fn test_signed_compressed_blob_writer() -> Result<(), Error> {
#[test]
fn test_encrypted_blob_writer() -> Result<(), Error> {
let tmp = Cursor::new(Vec::<u8>::new());
let mut blob_writer = DataBlobWriter::new_encrypted(tmp, &CRYPT_CONFIG)?;
let mut blob_writer = DataBlobWriter::new_encrypted(tmp, CRYPT_CONFIG.clone())?;
blob_writer.write_all(&TEST_DATA)?;
verify_test_blob(blob_writer.finish()?)
@ -107,7 +108,7 @@ fn test_encrypted_blob_writer() -> Result<(), Error> {
#[test]
fn test_encrypted_compressed_blob_writer() -> Result<(), Error> {
let tmp = Cursor::new(Vec::<u8>::new());
let mut blob_writer = DataBlobWriter::new_encrypted_compressed(tmp, &CRYPT_CONFIG)?;
let mut blob_writer = DataBlobWriter::new_encrypted_compressed(tmp, CRYPT_CONFIG.clone())?;
blob_writer.write_all(&TEST_DATA)?;
verify_test_blob(blob_writer.finish()?)