src/backup/data_blob.rs: new file format for binary blobs
This commit is contained in:
parent
077a8cae63
commit
3025b3a53c
@ -121,6 +121,18 @@ pub static COMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [191, 237, 46, 195, 108, 17, 22
|
||||
// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted chunk v1.0")[0..8]
|
||||
pub static ENCR_COMPR_CHUNK_MAGIC_1_0: [u8; 8] = [9, 40, 53, 200, 37, 150, 90, 196];
|
||||
|
||||
// openssl::sha::sha256(b"Proxmox Backup uncompressed blob v1.0")[0..8]
|
||||
pub static UNCOMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [66, 171, 56, 7, 190, 131, 112, 161];
|
||||
|
||||
//openssl::sha::sha256(b"Proxmox Backup zstd compressed blob v1.0")[0..8]
|
||||
pub static COMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [49, 185, 88, 66, 111, 182, 163, 127];
|
||||
|
||||
// openssl::sha::sha256(b"Proxmox Backup encrypted blob v1.0")[0..8]
|
||||
pub static ENCRYPTED_BLOB_MAGIC_1_0: [u8; 8] = [123, 103, 133, 190, 34, 45, 76, 240];
|
||||
|
||||
// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted blob v1.0")[0..8]
|
||||
pub static ENCR_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [230, 89, 27, 191, 11, 191, 216, 11];
|
||||
|
||||
// openssl::sha::sha256(b"Proxmox Backup fixed sized chunk index v1.0")[0..8]
|
||||
pub static FIXED_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [47, 127, 65, 237, 145, 253, 15, 205];
|
||||
|
||||
@ -136,6 +148,9 @@ pub use key_derivation::*;
|
||||
mod data_chunk;
|
||||
pub use data_chunk::*;
|
||||
|
||||
mod data_blob;
|
||||
pub use data_blob::*;
|
||||
|
||||
mod chunk_stream;
|
||||
pub use chunk_stream::*;
|
||||
|
||||
|
107
src/backup/data_blob.rs
Normal file
107
src/backup/data_blob.rs
Normal file
@ -0,0 +1,107 @@
|
||||
use failure::*;
|
||||
use std::convert::TryInto;
|
||||
use std::io::Write;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Data blob binary storage format
|
||||
///
|
||||
/// Data blobs store arbitrary binary data (< 16MB), and can be
|
||||
/// compressed and encrypted. A simply binary format is used to store
|
||||
/// them on disk or transfer them over the network. Please use index
|
||||
/// files to store large data files (".fidx" of ".didx").
|
||||
///
|
||||
/// The format start with a 8 byte magic number to identify the type.
|
||||
/// Encrypted blobs contain a 16 byte IV, followed by a 18 byte AD
|
||||
/// tag, followed by the encrypted data (MAGIC || IV || TAG ||
|
||||
/// EncryptedData).
|
||||
///
|
||||
/// Unencrypted blobs simply contain the (compressed) data.
|
||||
///
|
||||
/// This is basically the same format we use for ``DataChunk``, but
|
||||
/// with other magic numbers so that we can distinguish them.
|
||||
pub struct DataBlob {
|
||||
raw_data: Vec<u8>, // tagged, compressed, encryped data
|
||||
}
|
||||
|
||||
impl DataBlob {
|
||||
|
||||
/// accessor to raw_data field
|
||||
pub fn raw_data(&self) -> &[u8] {
|
||||
&self.raw_data
|
||||
}
|
||||
|
||||
/// accessor to chunk type (magic number)
|
||||
pub fn magic(&self) -> &[u8; 8] {
|
||||
self.raw_data[0..8].try_into().unwrap()
|
||||
}
|
||||
|
||||
pub fn encode(
|
||||
data: &[u8],
|
||||
config: Option<&CryptConfig>,
|
||||
compress: bool,
|
||||
) -> Result<Self, Error> {
|
||||
|
||||
if data.len() > 16*1024*1024 {
|
||||
bail!("data blob too large ({} bytes).", data.len());
|
||||
}
|
||||
|
||||
if let Some(config) = config {
|
||||
|
||||
let enc_data = config.encode_chunk(
|
||||
data,
|
||||
compress,
|
||||
&ENCRYPTED_BLOB_MAGIC_1_0,
|
||||
&ENCR_COMPR_BLOB_MAGIC_1_0,
|
||||
)?;
|
||||
return Ok(DataBlob { raw_data: enc_data });
|
||||
} else {
|
||||
|
||||
if compress {
|
||||
let mut comp_data = Vec::with_capacity(data.len() + 8);
|
||||
|
||||
comp_data.write_all(&COMPRESSED_BLOB_MAGIC_1_0)?;
|
||||
zstd::stream::copy_encode(data, &mut comp_data, 1)?;
|
||||
|
||||
if comp_data.len() < (data.len() + 8) {
|
||||
return Ok(DataBlob { raw_data: comp_data });
|
||||
}
|
||||
}
|
||||
|
||||
let mut raw_data = Vec::with_capacity(data.len() + 8);
|
||||
|
||||
raw_data.write_all(&UNCOMPRESSED_BLOB_MAGIC_1_0)?;
|
||||
raw_data.extend_from_slice(data);
|
||||
|
||||
return Ok(DataBlob { raw_data });
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode blob data
|
||||
pub fn decode(self, config: Option<&CryptConfig>) -> Result<Vec<u8>, Error> {
|
||||
|
||||
let magic = self.magic();
|
||||
|
||||
if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
|
||||
return Ok(self.raw_data);
|
||||
} else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
|
||||
|
||||
let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?;
|
||||
return Ok(data);
|
||||
|
||||
} else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
|
||||
if let Some(config) = config {
|
||||
let data = if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 {
|
||||
config.decode_compressed_chunk(&self.raw_data)?
|
||||
} else {
|
||||
config.decode_uncompressed_chunk(&self.raw_data)?
|
||||
};
|
||||
return Ok(data);
|
||||
} else {
|
||||
bail!("unable to decrypt blob - missing CryptConfig");
|
||||
}
|
||||
} else {
|
||||
bail!("Invalid blob magic number.");
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user