tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add crc field for binary blobs formats

Browse Source
This commit is contained in:
Dietmar Maurer 2019-06-21 17:24:21 +02:00
parent a7f67a9a9c
commit b7f4f27d6c
3 changed files with 104 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/backup/crypt_config.rs
View File

@ -85,31 +85,33 @@ impl CryptConfig {
let compr_data = zstd::block::compress(data, 1)?; let compr_data = zstd::block::compress(data, 1)?;
// Note: We only use compression if result is shorter // Note: We only use compression if result is shorter
if compr_data.len() < data.len() { if compr_data.len() < data.len() {
let mut enc = vec![0; compr_data.len()+40+self.cipher.block_size()]; let mut enc = vec![0; compr_data.len()+44+self.cipher.block_size()];
enc[0..8].copy_from_slice(comp_magic); enc[0..8].copy_from_slice(comp_magic);
enc[8..24].copy_from_slice(&iv); enc[8..12].copy_from_slice(&[0u8; 4]);
enc[12..28].copy_from_slice(&iv);
let count = c.update(&compr_data, &mut enc[40..])?; let count = c.update(&compr_data, &mut enc[44..])?;
let rest = c.finalize(&mut enc[(40+count)..])?; let rest = c.finalize(&mut enc[(44+count)..])?;
enc.truncate(40 + count + rest); enc.truncate(44 + count + rest);
c.get_tag(&mut enc[24..40])?; c.get_tag(&mut enc[28..44])?;
return Ok(enc) return Ok(enc)
} }
} }
let mut enc = vec![0; data.len()+40+self.cipher.block_size()]; let mut enc = vec![0; data.len()+44+self.cipher.block_size()];
enc[0..8].copy_from_slice(uncomp_magic); enc[0..8].copy_from_slice(uncomp_magic);
enc[8..24].copy_from_slice(&iv); enc[8..12].copy_from_slice(&[0u8; 4]);
enc[12..28].copy_from_slice(&iv);
let count = c.update(data, &mut enc[40..])?; let count = c.update(data, &mut enc[44..])?;
let rest = c.finalize(&mut enc[(40+count)..])?; let rest = c.finalize(&mut enc[(44+count)..])?;
enc.truncate(40 + count + rest); enc.truncate(44 + count + rest);
c.get_tag(&mut enc[24..40])?; c.get_tag(&mut enc[28..44])?;
Ok(enc) Ok(enc)
} }
@ -120,13 +122,14 @@ impl CryptConfig {
/// is not used here. /// is not used here.
pub fn decode_compressed_chunk(&self, data: &[u8]) -> Result<Vec<u8>, Error> { pub fn decode_compressed_chunk(&self, data: &[u8]) -> Result<Vec<u8>, Error> {
if data.len() < 40 { if data.len() < 44 {
bail!("Invalid chunk len (<40)"); bail!("Invalid chunk len (<44)");
} }
// let magic = &data[0..8]; // let magic = &data[0..8];
let iv = &data[8..24]; // let crc = &data[8..12];
let mac = &data[24..40]; let iv = &data[12..28];
let mac = &data[28..44];
let dec = Vec::with_capacity(1024*1024); let dec = Vec::with_capacity(1024*1024);
@ -140,7 +143,7 @@ impl CryptConfig {
let mut decr_buf = [0u8; BUFFER_SIZE]; let mut decr_buf = [0u8; BUFFER_SIZE];
let max_decoder_input = BUFFER_SIZE - self.cipher.block_size(); let max_decoder_input = BUFFER_SIZE - self.cipher.block_size();
let mut start = 40; let mut start = 44;
loop { loop {
let mut end = start + max_decoder_input; let mut end = start + max_decoder_input;
if end > data.len() { end = data.len(); } if end > data.len() { end = data.len(); }
@ -168,20 +171,21 @@ impl CryptConfig {
/// is not used here. /// is not used here.
pub fn decode_uncompressed_chunk(&self, data: &[u8]) -> Result<Vec<u8>, Error> { pub fn decode_uncompressed_chunk(&self, data: &[u8]) -> Result<Vec<u8>, Error> {
if data.len() < 40 { if data.len() < 44 {
bail!("Invalid chunk len (<40)"); bail!("Invalid chunk len (<44)");
} }
// let magic = &data[0..8]; // let magic = &data[0..8];
let iv = &data[8..24]; // let crc = &data[8..12];
let mac = &data[24..40]; let iv = &data[12..28];
let mac = &data[28..44];
let decr_data = decrypt_aead( let decr_data = decrypt_aead(
self.cipher, self.cipher,
&self.enc_key, &self.enc_key,
Some(iv), Some(iv),
b"", //?? b"", //??
&data[40..], &data[44..],
mac, mac,
)?; )?;

48
src/backup/data_blob.rs
View File

@ -11,12 +11,20 @@ use super::*;
/// them on disk or transfer them over the network. Please use index /// them on disk or transfer them over the network. Please use index
/// files to store large data files (".fidx" of ".didx"). /// files to store large data files (".fidx" of ".didx").
/// ///
/// The format start with a 8 byte magic number to identify the type. /// The format start with a 8 byte magic number to identify the type,
/// Encrypted blobs contain a 16 byte IV, followed by a 18 byte AD /// followed by a 4 byte CRC. This CRC is used on the server side to
/// tag, followed by the encrypted data (MAGIC || IV || TAG || /// detect file corruption (computed when upload data), so there is
/// EncryptedData). /// usually no need to compute it on the client side.
/// ///
/// Unencrypted blobs simply contain the (compressed) data. /// Encrypted blobs contain a 16 byte IV, followed by a 16 byte AD
/// tag, followed by the encrypted data:
///
/// (MAGIC || CRC32 || IV || TAG || EncryptedData).
///
/// Unencrypted blobs simply contain the CRC, followed by the
/// (compressed) data.
///
/// (MAGIC || CRC32 || Data)
/// ///
/// This is basically the same format we use for ``DataChunk``, but /// This is basically the same format we use for ``DataChunk``, but
/// with other magic numbers so that we can distinguish them. /// with other magic numbers so that we can distinguish them.
@ -36,6 +44,23 @@ impl DataBlob {
self.raw_data[0..8].try_into().unwrap() self.raw_data[0..8].try_into().unwrap()
} }
/// accessor to crc32 checksum
pub fn crc(&self) -> u32 {
u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap())
}
// set the CRC checksum field
pub fn set_crc(&mut self, crc: u32) {
self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes());
}
/// compute the CRC32 checksum
pub fn compute_crc(&mut self) -> u32 {
let mut hasher = crc32fast::Hasher::new();
hasher.update(&self.raw_data[12..]);
hasher.finalize()
}
pub fn encode( pub fn encode(
data: &[u8], data: &[u8],
config: Option<&CryptConfig>, config: Option<&CryptConfig>,
@ -58,19 +83,22 @@ impl DataBlob {
} else { } else {
if compress { if compress {
let mut comp_data = Vec::with_capacity(data.len() + 8); let mut comp_data = Vec::with_capacity(data.len() + 8 + 4);
comp_data.write_all(&COMPRESSED_BLOB_MAGIC_1_0)?; comp_data.write_all(&COMPRESSED_BLOB_MAGIC_1_0)?;
comp_data.write_all(&[0u8, 4])?; // CRC set to 0
zstd::stream::copy_encode(data, &mut comp_data, 1)?; zstd::stream::copy_encode(data, &mut comp_data, 1)?;
if comp_data.len() < (data.len() + 8) { if comp_data.len() < (data.len() + 8 + 4) {
return Ok(DataBlob { raw_data: comp_data }); return Ok(DataBlob { raw_data: comp_data });
} }
} }
let mut raw_data = Vec::with_capacity(data.len() + 8); let mut raw_data = Vec::with_capacity(data.len() + 8 + 4);
raw_data.write_all(&UNCOMPRESSED_BLOB_MAGIC_1_0)?; raw_data.write_all(&UNCOMPRESSED_BLOB_MAGIC_1_0)?;
raw_data.write_all(&[0u8; 4])?;
raw_data.extend_from_slice(data); raw_data.extend_from_slice(data);
return Ok(DataBlob { raw_data }); return Ok(DataBlob { raw_data });
@ -83,10 +111,10 @@ impl DataBlob {
let magic = self.magic(); let magic = self.magic();
if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 { if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
return Ok(self.raw_data); return Ok(self.raw_data[12..].to_vec());
} else if magic == &COMPRESSED_BLOB_MAGIC_1_0 { } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?; let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
return Ok(data); return Ok(data);
} else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 { } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {

61
src/backup/data_chunk.rs
View File

@ -17,12 +17,20 @@ pub struct ChunkInfo {
/// compressed and encrypted. A simply binary format is used to store /// compressed and encrypted. A simply binary format is used to store
/// them on disk or transfer them over the network. /// them on disk or transfer them over the network.
/// ///
/// The format start with a 8 byte magic number to identify the type. /// The format start with a 8 byte magic number to identify the type,
/// Encrypted chunks contain a 16 byte IV, followed by a 18 byte AD /// followed by a 4 byte CRC. This CRC is used on the server side to
/// tag, followed by the encrypted data (MAGIC || IV || TAG || /// detect file corruption (computed when upload data), so there is
/// EncryptedData). /// usually no need to compute it on the client side.
/// ///
/// Unecrypted chunks simply contain the (compressed) data. /// Encrypted chunks contain a 16 byte IV, followed by a 16 byte AD
/// tag, followed by the encrypted data:
///
/// (MAGIC || CRC32 || IV || TAG || EncryptedData).
///
/// Unencrypted blobs simply contain the CRC, followed by the
/// (compressed) data.
///
/// (MAGIC || CRC32 || Data)
/// ///
/// Please use the ``DataChunkBuilder`` to create new instances. /// Please use the ``DataChunkBuilder`` to create new instances.
pub struct DataChunk { pub struct DataChunk {
@ -47,15 +55,22 @@ impl DataChunk {
self.raw_data[0..8].try_into().unwrap() self.raw_data[0..8].try_into().unwrap()
} }
// only valid for enrypted data /// accessor to crc32 checksum
//pub fn iv(&self) -> &[u8; 16] { pub fn crc(&self) -> u32 {
// self.raw_data[8..24].try_into().unwrap() u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap())
//} }
// only valid for enrypted data // set the CRC checksum field
//pub fn mac(&self) -> &[u8; 16] { pub fn set_crc(&mut self, crc: u32) {
// self.raw_data[24..40].try_into().unwrap() self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes());
//} }
/// compute the CRC32 checksum
pub fn compute_crc(&mut self) -> u32 {
let mut hasher = crc32fast::Hasher::new();
hasher.update(&self.raw_data[12..]);
hasher.finalize()
}
fn new( fn new(
data: &[u8], data: &[u8],
@ -78,20 +93,22 @@ impl DataChunk {
} else { } else {
if compress { if compress {
let mut comp_data = Vec::with_capacity(data.len() + 8); let mut comp_data = Vec::with_capacity(data.len() + 8 + 4);
comp_data.write_all(&COMPRESSED_CHUNK_MAGIC_1_0)?; comp_data.write_all(&COMPRESSED_CHUNK_MAGIC_1_0)?;
comp_data.write_all(&[0u8, 4])?; // CRC set to 0
zstd::stream::copy_encode(data, &mut comp_data, 1)?; zstd::stream::copy_encode(data, &mut comp_data, 1)?;
if comp_data.len() < (data.len() + 8) { if comp_data.len() < (data.len() + 8 + 4) {
let chunk = DataChunk { digest, raw_data: comp_data }; let chunk = DataChunk { digest, raw_data: comp_data };
return Ok(chunk); return Ok(chunk);
} }
} }
let mut raw_data = Vec::with_capacity(data.len() + 8); let mut raw_data = Vec::with_capacity(data.len() + 8 + 4);
raw_data.write_all(&UNCOMPRESSED_CHUNK_MAGIC_1_0)?; raw_data.write_all(&UNCOMPRESSED_CHUNK_MAGIC_1_0)?;
raw_data.write_all(&[0u8, 4])?; // CRC set to 0
raw_data.extend_from_slice(data); raw_data.extend_from_slice(data);
let chunk = DataChunk { digest, raw_data }; let chunk = DataChunk { digest, raw_data };
@ -105,10 +122,10 @@ impl DataChunk {
let magic = self.magic(); let magic = self.magic();
if magic == &UNCOMPRESSED_CHUNK_MAGIC_1_0 { if magic == &UNCOMPRESSED_CHUNK_MAGIC_1_0 {
return Ok(self.raw_data); return Ok(self.raw_data[12..].to_vec());
} else if magic == &COMPRESSED_CHUNK_MAGIC_1_0 { } else if magic == &COMPRESSED_CHUNK_MAGIC_1_0 {
let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?; let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
return Ok(data); return Ok(data);
} else if magic == &ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == &ENCRYPTED_CHUNK_MAGIC_1_0 { } else if magic == &ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == &ENCRYPTED_CHUNK_MAGIC_1_0 {
@ -143,7 +160,7 @@ impl DataChunk {
/// Create Instance from raw data /// Create Instance from raw data
pub fn from_raw(data: Vec<u8>, digest: [u8;32]) -> Result<Self, Error> { pub fn from_raw(data: Vec<u8>, digest: [u8;32]) -> Result<Self, Error> {
if data.len() < 8 { if data.len() < 12 {
bail!("chunk too small ({} bytes).", data.len()); bail!("chunk too small ({} bytes).", data.len());
} }
@ -151,7 +168,7 @@ impl DataChunk {
if magic == ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == ENCRYPTED_CHUNK_MAGIC_1_0 { if magic == ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == ENCRYPTED_CHUNK_MAGIC_1_0 {
if data.len() < 40 { if data.len() < 44 {
bail!("encrypted chunk too small ({} bytes).", data.len()); bail!("encrypted chunk too small ({} bytes).", data.len());
} }
@ -188,10 +205,10 @@ impl DataChunk {
}; };
if magic == COMPRESSED_CHUNK_MAGIC_1_0 { if magic == COMPRESSED_CHUNK_MAGIC_1_0 {
let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?; let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
verify_raw_data(&data)?; verify_raw_data(&data)?;
} else if magic == UNCOMPRESSED_CHUNK_MAGIC_1_0 { } else if magic == UNCOMPRESSED_CHUNK_MAGIC_1_0 {
verify_raw_data(&self.raw_data[8..])?; verify_raw_data(&self.raw_data[12..])?;
} }
Ok(()) Ok(())