add crc field for binary blobs formats

src/backup/data_blob.rs

@@ -11,12 +11,20 @@ use super::*;
 /// them on disk or transfer them over the network. Please use index
 /// files to store large data files (".fidx" of ".didx").
 ///
-/// The format start with a 8 byte magic number to identify the type.
-/// Encrypted blobs contain a 16 byte IV, followed by a 18 byte AD
-/// tag, followed by the encrypted data (MAGIC || IV || TAG ||
-/// EncryptedData).
+/// The format start with a 8 byte magic number to identify the type,
+/// followed by a 4 byte CRC. This CRC is used on the server side to
+/// detect file corruption (computed when upload data), so there is
+/// usually no need to compute it on the client side.
 ///
-/// Unencrypted blobs simply contain the (compressed) data.
+/// Encrypted blobs contain a 16 byte IV, followed by a 16 byte AD
+/// tag, followed by the encrypted data:
+///
+/// (MAGIC || CRC32 || IV || TAG || EncryptedData).
+///
+/// Unencrypted blobs simply contain the CRC, followed by the
+/// (compressed) data.
+///
+/// (MAGIC || CRC32 || Data)
 ///
 /// This is basically the same format we use for ``DataChunk``, but
 /// with other magic numbers so that we can distinguish them.
@@ -36,6 +44,23 @@ impl DataBlob {
         self.raw_data[0..8].try_into().unwrap()
     }
 
+    /// accessor to crc32 checksum
+    pub fn crc(&self) -> u32 {
+        u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap())
+    }
+
+    // set the CRC checksum field
+    pub fn set_crc(&mut self, crc: u32) {
+        self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes());
+    }
+
+    /// compute the CRC32 checksum
+    pub fn compute_crc(&mut self) -> u32 {
+        let mut hasher = crc32fast::Hasher::new();
+        hasher.update(&self.raw_data[12..]);
+        hasher.finalize()
+    }
+
     pub fn encode(
         data: &[u8],
         config: Option<&CryptConfig>,
@@ -58,19 +83,22 @@ impl DataBlob {
         } else {
 
             if compress {
-                let mut comp_data = Vec::with_capacity(data.len() + 8);
+                let mut comp_data = Vec::with_capacity(data.len() + 8 + 4);
 
                 comp_data.write_all(&COMPRESSED_BLOB_MAGIC_1_0)?;
+                comp_data.write_all(&[0u8, 4])?; // CRC set to 0
+
                 zstd::stream::copy_encode(data, &mut comp_data, 1)?;
 
-                if comp_data.len() < (data.len() + 8) {
+                if comp_data.len() < (data.len() + 8 + 4) {
                     return Ok(DataBlob { raw_data: comp_data });
                 }
             }
 
-            let mut raw_data = Vec::with_capacity(data.len() + 8);
+            let mut raw_data = Vec::with_capacity(data.len() + 8 + 4);
 
             raw_data.write_all(&UNCOMPRESSED_BLOB_MAGIC_1_0)?;
+            raw_data.write_all(&[0u8; 4])?;
             raw_data.extend_from_slice(data);
 
             return Ok(DataBlob { raw_data });
@@ -83,10 +111,10 @@ impl DataBlob {
         let magic = self.magic();
 
         if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
-             return Ok(self.raw_data);
+            return Ok(self.raw_data[12..].to_vec());
         } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
 
-            let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?;
+            let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
             return Ok(data);
 
         } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {