tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add crc field for binary blobs formats

Browse Source
This commit is contained in:
Dietmar Maurer 2019-06-21 17:24:21 +02:00
parent a7f67a9a9c
commit b7f4f27d6c
3 changed files with 104 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/backup/crypt_config.rs
View File

@ -82,34 +82,36 @@ impl CryptConfig {
c.aad_update(b"")?; //??
if compress {
let compr_data = zstd::block::compress(data, 1)?;
let compr_data = zstd::block::compress(data, 1)?;
// Note: We only use compression if result is shorter
if compr_data.len() < data.len() {
let mut enc = vec![0; compr_data.len()+40+self.cipher.block_size()];
let mut enc = vec![0; compr_data.len()+44+self.cipher.block_size()];
enc[0..8].copy_from_slice(comp_magic);
enc[8..24].copy_from_slice(&iv);
enc[8..12].copy_from_slice(&[0u8; 4]);
enc[12..28].copy_from_slice(&iv);
let count = c.update(&compr_data, &mut enc[40..])?;
let rest = c.finalize(&mut enc[(40+count)..])?;
enc.truncate(40 + count + rest);
let count = c.update(&compr_data, &mut enc[44..])?;
let rest = c.finalize(&mut enc[(44+count)..])?;
enc.truncate(44 + count + rest);
c.get_tag(&mut enc[24..40])?;
c.get_tag(&mut enc[28..44])?;
return Ok(enc)
}
}
let mut enc = vec![0; data.len()+40+self.cipher.block_size()];
let mut enc = vec![0; data.len()+44+self.cipher.block_size()];
enc[0..8].copy_from_slice(uncomp_magic);
enc[8..24].copy_from_slice(&iv);
enc[8..12].copy_from_slice(&[0u8; 4]);
enc[12..28].copy_from_slice(&iv);
let count = c.update(data, &mut enc[40..])?;
let rest = c.finalize(&mut enc[(40+count)..])?;
enc.truncate(40 + count + rest);
let count = c.update(data, &mut enc[44..])?;
let rest = c.finalize(&mut enc[(44+count)..])?;
enc.truncate(44 + count + rest);
c.get_tag(&mut enc[24..40])?;
c.get_tag(&mut enc[28..44])?;
Ok(enc)
}
@ -120,13 +122,14 @@ impl CryptConfig {
/// is not used here.
pub fn decode_compressed_chunk(&self, data: &[u8]) -> Result<Vec<u8>, Error> {
if data.len() < 40 {
bail!("Invalid chunk len (<40)");
if data.len() < 44 {
bail!("Invalid chunk len (<44)");
}
// let magic = &data[0..8];
let iv = &data[8..24];
let mac = &data[24..40];
// let crc = &data[8..12];
let iv = &data[12..28];
let mac = &data[28..44];
let dec = Vec::with_capacity(1024*1024);
@ -140,7 +143,7 @@ impl CryptConfig {
let mut decr_buf = [0u8; BUFFER_SIZE];
let max_decoder_input = BUFFER_SIZE - self.cipher.block_size();
let mut start = 40;
let mut start = 44;
loop {
let mut end = start + max_decoder_input;
if end > data.len() { end = data.len(); }
@ -168,20 +171,21 @@ impl CryptConfig {
/// is not used here.
pub fn decode_uncompressed_chunk(&self, data: &[u8]) -> Result<Vec<u8>, Error> {
if data.len() < 40 {
bail!("Invalid chunk len (<40)");
if data.len() < 44 {
bail!("Invalid chunk len (<44)");
}
// let magic = &data[0..8];
let iv = &data[8..24];
let mac = &data[24..40];
// let crc = &data[8..12];
let iv = &data[12..28];
let mac = &data[28..44];
let decr_data = decrypt_aead(
self.cipher,
&self.enc_key,
Some(iv),
b"", //??
&data[40..],
&data[44..],
mac,
)?;

48
src/backup/data_blob.rs
View File

@ -11,12 +11,20 @@ use super::*;
/// them on disk or transfer them over the network. Please use index
/// files to store large data files (".fidx" of ".didx").
///
/// The format start with a 8 byte magic number to identify the type.
/// Encrypted blobs contain a 16 byte IV, followed by a 18 byte AD
/// tag, followed by the encrypted data (MAGIC || IV || TAG ||
/// EncryptedData).
/// The format start with a 8 byte magic number to identify the type,
/// followed by a 4 byte CRC. This CRC is used on the server side to
/// detect file corruption (computed when upload data), so there is
/// usually no need to compute it on the client side.
///
/// Unencrypted blobs simply contain the (compressed) data.
/// Encrypted blobs contain a 16 byte IV, followed by a 16 byte AD
/// tag, followed by the encrypted data:
///
/// (MAGIC || CRC32 || IV || TAG || EncryptedData).
///
/// Unencrypted blobs simply contain the CRC, followed by the
/// (compressed) data.
///
/// (MAGIC || CRC32 || Data)
///
/// This is basically the same format we use for ``DataChunk``, but
/// with other magic numbers so that we can distinguish them.
@ -36,6 +44,23 @@ impl DataBlob {
self.raw_data[0..8].try_into().unwrap()
}
/// accessor to crc32 checksum
pub fn crc(&self) -> u32 {
u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap())
}
// set the CRC checksum field
pub fn set_crc(&mut self, crc: u32) {
self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes());
}
/// compute the CRC32 checksum
pub fn compute_crc(&mut self) -> u32 {
let mut hasher = crc32fast::Hasher::new();
hasher.update(&self.raw_data[12..]);
hasher.finalize()
}
pub fn encode(
data: &[u8],
config: Option<&CryptConfig>,
@ -58,19 +83,22 @@ impl DataBlob {
} else {
if compress {
let mut comp_data = Vec::with_capacity(data.len() + 8);
let mut comp_data = Vec::with_capacity(data.len() + 8 + 4);
comp_data.write_all(&COMPRESSED_BLOB_MAGIC_1_0)?;
comp_data.write_all(&[0u8, 4])?; // CRC set to 0
zstd::stream::copy_encode(data, &mut comp_data, 1)?;
if comp_data.len() < (data.len() + 8) {
if comp_data.len() < (data.len() + 8 + 4) {
return Ok(DataBlob { raw_data: comp_data });
}
}
let mut raw_data = Vec::with_capacity(data.len() + 8);
let mut raw_data = Vec::with_capacity(data.len() + 8 + 4);
raw_data.write_all(&UNCOMPRESSED_BLOB_MAGIC_1_0)?;
raw_data.write_all(&[0u8; 4])?;
raw_data.extend_from_slice(data);
return Ok(DataBlob { raw_data });
@ -83,10 +111,10 @@ impl DataBlob {
let magic = self.magic();
if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
return Ok(self.raw_data);
return Ok(self.raw_data[12..].to_vec());
} else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?;
let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
return Ok(data);
} else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {

61
src/backup/data_chunk.rs
View File

@ -17,12 +17,20 @@ pub struct ChunkInfo {
/// compressed and encrypted. A simply binary format is used to store
/// them on disk or transfer them over the network.
///
/// The format start with a 8 byte magic number to identify the type.
/// Encrypted chunks contain a 16 byte IV, followed by a 18 byte AD
/// tag, followed by the encrypted data (MAGIC || IV || TAG ||
/// EncryptedData).
/// The format start with a 8 byte magic number to identify the type,
/// followed by a 4 byte CRC. This CRC is used on the server side to
/// detect file corruption (computed when upload data), so there is
/// usually no need to compute it on the client side.
///
/// Unecrypted chunks simply contain the (compressed) data.
/// Encrypted chunks contain a 16 byte IV, followed by a 16 byte AD
/// tag, followed by the encrypted data:
///
/// (MAGIC || CRC32 || IV || TAG || EncryptedData).
///
/// Unencrypted blobs simply contain the CRC, followed by the
/// (compressed) data.
///
/// (MAGIC || CRC32 || Data)
///
/// Please use the ``DataChunkBuilder`` to create new instances.
pub struct DataChunk {
@ -47,15 +55,22 @@ impl DataChunk {
self.raw_data[0..8].try_into().unwrap()
}
// only valid for enrypted data
//pub fn iv(&self) -> &[u8; 16] {
// self.raw_data[8..24].try_into().unwrap()
//}
/// accessor to crc32 checksum
pub fn crc(&self) -> u32 {
u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap())
}
// only valid for enrypted data
//pub fn mac(&self) -> &[u8; 16] {
// self.raw_data[24..40].try_into().unwrap()
//}
// set the CRC checksum field
pub fn set_crc(&mut self, crc: u32) {
self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes());
}
/// compute the CRC32 checksum
pub fn compute_crc(&mut self) -> u32 {
let mut hasher = crc32fast::Hasher::new();
hasher.update(&self.raw_data[12..]);
hasher.finalize()
}
fn new(
data: &[u8],
@ -78,20 +93,22 @@ impl DataChunk {
} else {
if compress {
let mut comp_data = Vec::with_capacity(data.len() + 8);
let mut comp_data = Vec::with_capacity(data.len() + 8 + 4);
comp_data.write_all(&COMPRESSED_CHUNK_MAGIC_1_0)?;
comp_data.write_all(&[0u8, 4])?; // CRC set to 0
zstd::stream::copy_encode(data, &mut comp_data, 1)?;
if comp_data.len() < (data.len() + 8) {
if comp_data.len() < (data.len() + 8 + 4) {
let chunk = DataChunk { digest, raw_data: comp_data };
return Ok(chunk);
}
}
let mut raw_data = Vec::with_capacity(data.len() + 8);
let mut raw_data = Vec::with_capacity(data.len() + 8 + 4);
raw_data.write_all(&UNCOMPRESSED_CHUNK_MAGIC_1_0)?;
raw_data.write_all(&[0u8, 4])?; // CRC set to 0
raw_data.extend_from_slice(data);
let chunk = DataChunk { digest, raw_data };
@ -105,10 +122,10 @@ impl DataChunk {
let magic = self.magic();
if magic == &UNCOMPRESSED_CHUNK_MAGIC_1_0 {
return Ok(self.raw_data);
return Ok(self.raw_data[12..].to_vec());
} else if magic == &COMPRESSED_CHUNK_MAGIC_1_0 {
let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?;
let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
return Ok(data);
} else if magic == &ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == &ENCRYPTED_CHUNK_MAGIC_1_0 {
@ -143,7 +160,7 @@ impl DataChunk {
/// Create Instance from raw data
pub fn from_raw(data: Vec<u8>, digest: [u8;32]) -> Result<Self, Error> {
if data.len() < 8 {
if data.len() < 12 {
bail!("chunk too small ({} bytes).", data.len());
}
@ -151,7 +168,7 @@ impl DataChunk {
if magic == ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == ENCRYPTED_CHUNK_MAGIC_1_0 {
if data.len() < 40 {
if data.len() < 44 {
bail!("encrypted chunk too small ({} bytes).", data.len());
}
@ -188,10 +205,10 @@ impl DataChunk {
};
if magic == COMPRESSED_CHUNK_MAGIC_1_0 {
let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?;
let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
verify_raw_data(&data)?;
} else if magic == UNCOMPRESSED_CHUNK_MAGIC_1_0 {
verify_raw_data(&self.raw_data[8..])?;
verify_raw_data(&self.raw_data[12..])?;
}
Ok(())