src/backup/file_formats.rs: split out file format data

src/backup.rs

@@ -107,37 +107,8 @@ macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 {
     () =>  { "proxmox-backup-protocol-v1" }
 }
 
-// WARNING: PLEASE DO NOT MODIFY THOSE MAGIC VALUES
+mod file_formats;
-
+pub use file_formats::*;
-// openssl::sha::sha256(b"Proxmox Backup uncompressed chunk v1.0")[0..8]
-pub static UNCOMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [79, 127, 200, 4, 121, 74, 135, 239];
-
-// openssl::sha::sha256(b"Proxmox Backup encrypted chunk v1.0")[0..8]
-pub static ENCRYPTED_CHUNK_MAGIC_1_0: [u8; 8] = [8, 54, 114, 153, 70, 156, 26, 151];
-
-// openssl::sha::sha256(b"Proxmox Backup zstd compressed chunk v1.0")[0..8]
-pub static COMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [191, 237, 46, 195, 108, 17, 228, 235];
-
-// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted chunk v1.0")[0..8]
-pub static ENCR_COMPR_CHUNK_MAGIC_1_0: [u8; 8] = [9, 40, 53, 200, 37, 150, 90, 196];
-
-// openssl::sha::sha256(b"Proxmox Backup uncompressed blob v1.0")[0..8]
-pub static UNCOMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [66, 171, 56, 7, 190, 131, 112, 161];
-
-//openssl::sha::sha256(b"Proxmox Backup zstd compressed blob v1.0")[0..8]
-pub static COMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [49, 185, 88, 66, 111, 182, 163, 127];
-
-// openssl::sha::sha256(b"Proxmox Backup encrypted blob v1.0")[0..8]
-pub static ENCRYPTED_BLOB_MAGIC_1_0: [u8; 8] = [123, 103, 133, 190, 34, 45, 76, 240];
-
-// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted blob v1.0")[0..8]
-pub static ENCR_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [230, 89, 27, 191, 11, 191, 216, 11];
-
-// openssl::sha::sha256(b"Proxmox Backup fixed sized chunk index v1.0")[0..8]
-pub static FIXED_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [47, 127, 65, 237, 145, 253, 15, 205];
-
-// openssl::sha::sha256(b"Proxmox Backup dynamic sized chunk index v1.0")[0..8]
-pub static DYNAMIC_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [28, 145, 78, 165, 25, 186, 179, 205];
 
 mod crypt_config;
 pub use crypt_config::*;

src/backup/data_blob.rs

@@ -1,7 +1,7 @@
 use failure::*;
 use std::convert::TryInto;
-use std::io::Write;
 
+use crate::tools::write::*;
 use super::*;
 
 /// Data blob binary storage format
@@ -28,6 +28,7 @@ use super::*;
 ///
 /// This is basically the same format we use for ``DataChunk``, but
 /// with other magic numbers so that we can distinguish them.
+
 pub struct DataBlob {
     raw_data: Vec<u8>, // tagged, compressed, encryped data
 }
@@ -46,18 +47,21 @@ impl DataBlob {
 
     /// accessor to crc32 checksum
     pub fn crc(&self) -> u32 {
-        u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap())
+        let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc);
+        u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap())
     }
 
     // set the CRC checksum field
     pub fn set_crc(&mut self, crc: u32) {
-        self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes());
+        let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc);
+        self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes());
     }
 
     /// compute the CRC32 checksum
     pub fn compute_crc(&mut self) -> u32 {
         let mut hasher = crc32fast::Hasher::new();
-        hasher.update(&self.raw_data[12..]);
+        let start = std::mem::size_of::<DataBlobHeader>(); // start after HEAD
+        hasher.update(&self.raw_data[start..]);
         hasher.finalize()
     }
 
@@ -82,23 +86,30 @@ impl DataBlob {
             return Ok(DataBlob { raw_data: enc_data });
         } else {
 
+            let max_data_len = data.len() + std::mem::size_of::<DataBlobHeader>();
             if compress {
-                let mut comp_data = Vec::with_capacity(data.len() + 8 + 4);
+                let mut comp_data = Vec::with_capacity(max_data_len);
 
-                comp_data.write_all(&COMPRESSED_BLOB_MAGIC_1_0)?;
+                let head =  DataBlobHeader {
-                comp_data.write_all(&[0u8, 4])?; // CRC set to 0
+                    magic: COMPRESSED_BLOB_MAGIC_1_0,
+                    crc: [0; 4],
+                };
+                comp_data.write_value(&head)?;
 
                 zstd::stream::copy_encode(data, &mut comp_data, 1)?;
 
-                if comp_data.len() < (data.len() + 8 + 4) {
+                if comp_data.len() < max_data_len {
                     return Ok(DataBlob { raw_data: comp_data });
                 }
             }
 
-            let mut raw_data = Vec::with_capacity(data.len() + 8 + 4);
+            let mut raw_data = Vec::with_capacity(max_data_len);
 
-            raw_data.write_all(&UNCOMPRESSED_BLOB_MAGIC_1_0)?;
+            let head =  DataBlobHeader {
-            raw_data.write_all(&[0u8; 4])?;
+                magic: UNCOMPRESSED_BLOB_MAGIC_1_0,
+                crc: [0; 4],
+            };
+            raw_data.write_value(&head)?;
             raw_data.extend_from_slice(data);
 
             return Ok(DataBlob { raw_data });
@@ -111,12 +122,12 @@ impl DataBlob {
         let magic = self.magic();
 
         if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
-            return Ok(self.raw_data[12..].to_vec());
+            let data_start = std::mem::size_of::<DataBlobHeader>();
+            return Ok(self.raw_data[data_start..].to_vec());
         } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
-
+            let data_start = std::mem::size_of::<DataBlobHeader>();
-            let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
+            let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?;
             return Ok(data);
-
         } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
             if let Some(config) = config  {
                 let data = if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 {

src/backup/data_chunk.rs

@@ -1,6 +1,6 @@
 use failure::*;
 use std::convert::TryInto;
-use std::io::Write;
+use crate::tools::write::*;
 
 use super::*;
 
@@ -57,22 +57,25 @@ impl DataChunk {
 
         /// accessor to crc32 checksum
     pub fn crc(&self) -> u32 {
-        u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap())
+        let crc_o = proxmox::tools::offsetof!(DataChunkHeader, crc);
+        u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap())
     }
 
     // set the CRC checksum field
     pub fn set_crc(&mut self, crc: u32) {
-        self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes());
+        let crc_o = proxmox::tools::offsetof!(DataChunkHeader, crc);
+        self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes());
     }
 
     /// compute the CRC32 checksum
     pub fn compute_crc(&mut self) -> u32 {
         let mut hasher = crc32fast::Hasher::new();
-        hasher.update(&self.raw_data[12..]);
+        let start = std::mem::size_of::<DataChunkHeader>(); // start after HEAD
+        hasher.update(&self.raw_data[start..]);
         hasher.finalize()
     }
 
-    fn new(
+    fn encode(
         data: &[u8],
         config: Option<&CryptConfig>,
         digest: [u8;32],
@@ -92,23 +95,31 @@ impl DataChunk {
             Ok(chunk)
         } else {
 
+            let max_data_len = data.len() + std::mem::size_of::<DataChunkHeader>();
             if compress {
-                let mut comp_data = Vec::with_capacity(data.len() + 8 + 4);
+                let mut comp_data = Vec::with_capacity(max_data_len);
+
+                let head =  DataChunkHeader {
+                    magic: COMPRESSED_CHUNK_MAGIC_1_0,
+                    crc: [0; 4],
+                };
+                comp_data.write_value(&head)?;
 
-                comp_data.write_all(&COMPRESSED_CHUNK_MAGIC_1_0)?;
-                comp_data.write_all(&[0u8, 4])?; // CRC set to 0
                 zstd::stream::copy_encode(data, &mut comp_data, 1)?;
 
-                if comp_data.len() < (data.len() + 8 + 4) {
+                if comp_data.len() < max_data_len {
                     let chunk = DataChunk { digest, raw_data: comp_data };
                     return Ok(chunk);
                 }
             }
 
-            let mut raw_data = Vec::with_capacity(data.len() + 8 + 4);
+            let mut raw_data = Vec::with_capacity(max_data_len);
 
-            raw_data.write_all(&UNCOMPRESSED_CHUNK_MAGIC_1_0)?;
+            let head =  DataChunkHeader {
-            raw_data.write_all(&[0u8, 4])?; // CRC set to 0
+                magic: UNCOMPRESSED_CHUNK_MAGIC_1_0,
+                crc: [0; 4],
+            };
+            raw_data.write_value(&head)?;
             raw_data.extend_from_slice(data);
 
             let chunk = DataChunk { digest, raw_data };
@@ -122,12 +133,12 @@ impl DataChunk {
         let magic = self.magic();
 
         if magic == &UNCOMPRESSED_CHUNK_MAGIC_1_0 {
-            return Ok(self.raw_data[12..].to_vec());
+            let data_start = std::mem::size_of::<DataChunkHeader>();
+            return Ok(self.raw_data[data_start..].to_vec());
         } else if magic == &COMPRESSED_CHUNK_MAGIC_1_0 {
-
+            let data_start = std::mem::size_of::<DataChunkHeader>();
-            let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
+            let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?;
             return Ok(data);
-
         } else if magic == &ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == &ENCRYPTED_CHUNK_MAGIC_1_0 {
             if let Some(config) = config  {
                 let data = if magic == &ENCR_COMPR_CHUNK_MAGIC_1_0 {
@@ -160,7 +171,7 @@ impl DataChunk {
     /// Create Instance from raw data
     pub fn from_raw(data: Vec<u8>, digest: [u8;32]) -> Result<Self, Error> {
 
-        if data.len() < 12 {
+        if data.len() < std::mem::size_of::<DataChunkHeader>() {
             bail!("chunk too small ({} bytes).", data.len());
         }
 
@@ -168,7 +179,7 @@ impl DataChunk {
 
         if magic == ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == ENCRYPTED_CHUNK_MAGIC_1_0 {
 
-            if data.len() < 44 {
+            if data.len() < std::mem::size_of::<EncryptedDataChunkHeader>() {
                 bail!("encrypted chunk too small ({} bytes).", data.len());
             }
 
@@ -191,7 +202,7 @@ impl DataChunk {
     /// this is noth possible for encrypted chunks.
     pub fn verify_unencrypted(&self, expected_chunk_size: usize) -> Result<(), Error> {
 
-        let magic = &self.raw_data[0..8];
+        let magic = self.magic();
 
         let verify_raw_data = |data: &[u8]| {
             if expected_chunk_size != data.len() {
@@ -204,10 +215,10 @@ impl DataChunk {
             Ok(())
         };
 
-        if magic == COMPRESSED_CHUNK_MAGIC_1_0 {
+        if magic == &COMPRESSED_CHUNK_MAGIC_1_0 {
            let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
            verify_raw_data(&data)?;
-        } else if magic == UNCOMPRESSED_CHUNK_MAGIC_1_0 {
+        } else if magic == &UNCOMPRESSED_CHUNK_MAGIC_1_0 {
             verify_raw_data(&self.raw_data[12..])?;
         }
 
@@ -288,7 +299,12 @@ impl <'a, 'b> DataChunkBuilder<'a, 'b> {
             self.compute_digest();
         }
 
-        let chunk = DataChunk::new(self.orig_data, self.config, self.digest, self.compress)?;
+        let chunk = DataChunk::encode(
+            self.orig_data,
+            self.config,
+            self.digest,
+            self.compress,
+        )?;
 
         Ok(chunk)
     }

src/backup/file_formats.rs

@@ -0,0 +1,57 @@
+// WARNING: PLEASE DO NOT MODIFY THOSE MAGIC VALUES
+
+// openssl::sha::sha256(b"Proxmox Backup uncompressed chunk v1.0")[0..8]
+pub static UNCOMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [79, 127, 200, 4, 121, 74, 135, 239];
+
+// openssl::sha::sha256(b"Proxmox Backup encrypted chunk v1.0")[0..8]
+pub static ENCRYPTED_CHUNK_MAGIC_1_0: [u8; 8] = [8, 54, 114, 153, 70, 156, 26, 151];
+
+// openssl::sha::sha256(b"Proxmox Backup zstd compressed chunk v1.0")[0..8]
+pub static COMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [191, 237, 46, 195, 108, 17, 228, 235];
+
+// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted chunk v1.0")[0..8]
+pub static ENCR_COMPR_CHUNK_MAGIC_1_0: [u8; 8] = [9, 40, 53, 200, 37, 150, 90, 196];
+
+// openssl::sha::sha256(b"Proxmox Backup uncompressed blob v1.0")[0..8]
+pub static UNCOMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [66, 171, 56, 7, 190, 131, 112, 161];
+
+//openssl::sha::sha256(b"Proxmox Backup zstd compressed blob v1.0")[0..8]
+pub static COMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [49, 185, 88, 66, 111, 182, 163, 127];
+
+// openssl::sha::sha256(b"Proxmox Backup encrypted blob v1.0")[0..8]
+pub static ENCRYPTED_BLOB_MAGIC_1_0: [u8; 8] = [123, 103, 133, 190, 34, 45, 76, 240];
+
+// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted blob v1.0")[0..8]
+pub static ENCR_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [230, 89, 27, 191, 11, 191, 216, 11];
+
+// openssl::sha::sha256(b"Proxmox Backup fixed sized chunk index v1.0")[0..8]
+pub static FIXED_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [47, 127, 65, 237, 145, 253, 15, 205];
+
+// openssl::sha::sha256(b"Proxmox Backup dynamic sized chunk index v1.0")[0..8]
+pub static DYNAMIC_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [28, 145, 78, 165, 25, 186, 179, 205];
+
+#[repr(C,packed)]
+pub struct DataBlobHeader {
+    pub magic: [u8; 8],
+    pub crc: [u8; 4],
+}
+
+#[repr(C,packed)]
+pub struct EncryptedDataBlobHeader {
+    pub head: DataBlobHeader,
+    pub iv: [u8; 16],
+    pub tag: [u8; 16],
+}
+
+#[repr(C,packed)]
+pub struct DataChunkHeader {
+    pub magic: [u8; 8],
+    pub crc: [u8; 4],
+}
+
+#[repr(C,packed)]
+pub struct EncryptedDataChunkHeader {
+    pub head: DataChunkHeader,
+    pub iv: [u8; 16],
+    pub tag: [u8; 16],
+}