tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

src/backup/file_formats.rs: split out file format data

Browse Source
This commit is contained in:
Dietmar Maurer 2019-06-22 09:12:25 +02:00
parent b7f4f27d6c
commit 991abfa8b4
4 changed files with 123 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/backup.rs
View File

@ -107,37 +107,8 @@ macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 {
() => { "proxmox-backup-protocol-v1" }
}
// WARNING: PLEASE DO NOT MODIFY THOSE MAGIC VALUES
// openssl::sha::sha256(b"Proxmox Backup uncompressed chunk v1.0")[0..8]
pub static UNCOMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [79, 127, 200, 4, 121, 74, 135, 239];
// openssl::sha::sha256(b"Proxmox Backup encrypted chunk v1.0")[0..8]
pub static ENCRYPTED_CHUNK_MAGIC_1_0: [u8; 8] = [8, 54, 114, 153, 70, 156, 26, 151];
// openssl::sha::sha256(b"Proxmox Backup zstd compressed chunk v1.0")[0..8]
pub static COMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [191, 237, 46, 195, 108, 17, 228, 235];
// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted chunk v1.0")[0..8]
pub static ENCR_COMPR_CHUNK_MAGIC_1_0: [u8; 8] = [9, 40, 53, 200, 37, 150, 90, 196];
// openssl::sha::sha256(b"Proxmox Backup uncompressed blob v1.0")[0..8]
pub static UNCOMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [66, 171, 56, 7, 190, 131, 112, 161];
//openssl::sha::sha256(b"Proxmox Backup zstd compressed blob v1.0")[0..8]
pub static COMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [49, 185, 88, 66, 111, 182, 163, 127];
// openssl::sha::sha256(b"Proxmox Backup encrypted blob v1.0")[0..8]
pub static ENCRYPTED_BLOB_MAGIC_1_0: [u8; 8] = [123, 103, 133, 190, 34, 45, 76, 240];
// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted blob v1.0")[0..8]
pub static ENCR_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [230, 89, 27, 191, 11, 191, 216, 11];
// openssl::sha::sha256(b"Proxmox Backup fixed sized chunk index v1.0")[0..8]
pub static FIXED_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [47, 127, 65, 237, 145, 253, 15, 205];
// openssl::sha::sha256(b"Proxmox Backup dynamic sized chunk index v1.0")[0..8]
pub static DYNAMIC_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [28, 145, 78, 165, 25, 186, 179, 205];
mod file_formats;
pub use file_formats::*;
mod crypt_config;
pub use crypt_config::*;

41
src/backup/data_blob.rs
View File

@ -1,7 +1,7 @@
use failure::*;
use std::convert::TryInto;
use std::io::Write;
use crate::tools::write::*;
use super::*;
/// Data blob binary storage format
@ -28,6 +28,7 @@ use super::*;
///
/// This is basically the same format we use for ``DataChunk``, but
/// with other magic numbers so that we can distinguish them.
pub struct DataBlob {
raw_data: Vec<u8>, // tagged, compressed, encryped data
}
@ -46,18 +47,21 @@ impl DataBlob {
/// accessor to crc32 checksum
pub fn crc(&self) -> u32 {
u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap())
let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc);
u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap())
}
// set the CRC checksum field
pub fn set_crc(&mut self, crc: u32) {
self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes());
let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc);
self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes());
}
/// compute the CRC32 checksum
pub fn compute_crc(&mut self) -> u32 {
let mut hasher = crc32fast::Hasher::new();
hasher.update(&self.raw_data[12..]);
let start = std::mem::size_of::<DataBlobHeader>(); // start after HEAD
hasher.update(&self.raw_data[start..]);
hasher.finalize()
}
@ -82,23 +86,30 @@ impl DataBlob {
return Ok(DataBlob { raw_data: enc_data });
} else {
let max_data_len = data.len() + std::mem::size_of::<DataBlobHeader>();
if compress {
let mut comp_data = Vec::with_capacity(data.len() + 8 + 4);
let mut comp_data = Vec::with_capacity(max_data_len);
comp_data.write_all(&COMPRESSED_BLOB_MAGIC_1_0)?;
comp_data.write_all(&[0u8, 4])?; // CRC set to 0
let head = DataBlobHeader {
magic: COMPRESSED_BLOB_MAGIC_1_0,
crc: [0; 4],
};
comp_data.write_value(&head)?;
zstd::stream::copy_encode(data, &mut comp_data, 1)?;
if comp_data.len() < (data.len() + 8 + 4) {
if comp_data.len() < max_data_len {
return Ok(DataBlob { raw_data: comp_data });
}
}
let mut raw_data = Vec::with_capacity(data.len() + 8 + 4);
let mut raw_data = Vec::with_capacity(max_data_len);
raw_data.write_all(&UNCOMPRESSED_BLOB_MAGIC_1_0)?;
raw_data.write_all(&[0u8; 4])?;
let head = DataBlobHeader {
magic: UNCOMPRESSED_BLOB_MAGIC_1_0,
crc: [0; 4],
};
raw_data.write_value(&head)?;
raw_data.extend_from_slice(data);
return Ok(DataBlob { raw_data });
@ -111,12 +122,12 @@ impl DataBlob {
let magic = self.magic();
if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
return Ok(self.raw_data[12..].to_vec());
let data_start = std::mem::size_of::<DataBlobHeader>();
return Ok(self.raw_data[data_start..].to_vec());
} else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
let data_start = std::mem::size_of::<DataBlobHeader>();
let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?;
return Ok(data);
} else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
if let Some(config) = config {
let data = if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 {

60
src/backup/data_chunk.rs
View File

@ -1,6 +1,6 @@
use failure::*;
use std::convert::TryInto;
use std::io::Write;
use crate::tools::write::*;
use super::*;
@ -57,22 +57,25 @@ impl DataChunk {
/// accessor to crc32 checksum
pub fn crc(&self) -> u32 {
u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap())
let crc_o = proxmox::tools::offsetof!(DataChunkHeader, crc);
u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap())
}
// set the CRC checksum field
pub fn set_crc(&mut self, crc: u32) {
self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes());
let crc_o = proxmox::tools::offsetof!(DataChunkHeader, crc);
self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes());
}
/// compute the CRC32 checksum
pub fn compute_crc(&mut self) -> u32 {
let mut hasher = crc32fast::Hasher::new();
hasher.update(&self.raw_data[12..]);
let start = std::mem::size_of::<DataChunkHeader>(); // start after HEAD
hasher.update(&self.raw_data[start..]);
hasher.finalize()
}
fn new(
fn encode(
data: &[u8],
config: Option<&CryptConfig>,
digest: [u8;32],
@ -92,23 +95,31 @@ impl DataChunk {
Ok(chunk)
} else {
let max_data_len = data.len() + std::mem::size_of::<DataChunkHeader>();
if compress {
let mut comp_data = Vec::with_capacity(data.len() + 8 + 4);
let mut comp_data = Vec::with_capacity(max_data_len);
let head = DataChunkHeader {
magic: COMPRESSED_CHUNK_MAGIC_1_0,
crc: [0; 4],
};
comp_data.write_value(&head)?;
comp_data.write_all(&COMPRESSED_CHUNK_MAGIC_1_0)?;
comp_data.write_all(&[0u8, 4])?; // CRC set to 0
zstd::stream::copy_encode(data, &mut comp_data, 1)?;
if comp_data.len() < (data.len() + 8 + 4) {
if comp_data.len() < max_data_len {
let chunk = DataChunk { digest, raw_data: comp_data };
return Ok(chunk);
}
}
let mut raw_data = Vec::with_capacity(data.len() + 8 + 4);
let mut raw_data = Vec::with_capacity(max_data_len);
raw_data.write_all(&UNCOMPRESSED_CHUNK_MAGIC_1_0)?;
raw_data.write_all(&[0u8, 4])?; // CRC set to 0
let head = DataChunkHeader {
magic: UNCOMPRESSED_CHUNK_MAGIC_1_0,
crc: [0; 4],
};
raw_data.write_value(&head)?;
raw_data.extend_from_slice(data);
let chunk = DataChunk { digest, raw_data };
@ -122,12 +133,12 @@ impl DataChunk {
let magic = self.magic();
if magic == &UNCOMPRESSED_CHUNK_MAGIC_1_0 {
return Ok(self.raw_data[12..].to_vec());
let data_start = std::mem::size_of::<DataChunkHeader>();
return Ok(self.raw_data[data_start..].to_vec());
} else if magic == &COMPRESSED_CHUNK_MAGIC_1_0 {
let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
let data_start = std::mem::size_of::<DataChunkHeader>();
let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?;
return Ok(data);
} else if magic == &ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == &ENCRYPTED_CHUNK_MAGIC_1_0 {
if let Some(config) = config {
let data = if magic == &ENCR_COMPR_CHUNK_MAGIC_1_0 {
@ -160,7 +171,7 @@ impl DataChunk {
/// Create Instance from raw data
pub fn from_raw(data: Vec<u8>, digest: [u8;32]) -> Result<Self, Error> {
if data.len() < 12 {
if data.len() < std::mem::size_of::<DataChunkHeader>() {
bail!("chunk too small ({} bytes).", data.len());
}
@ -168,7 +179,7 @@ impl DataChunk {
if magic == ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == ENCRYPTED_CHUNK_MAGIC_1_0 {
if data.len() < 44 {
if data.len() < std::mem::size_of::<EncryptedDataChunkHeader>() {
bail!("encrypted chunk too small ({} bytes).", data.len());
}
@ -191,7 +202,7 @@ impl DataChunk {
/// this is noth possible for encrypted chunks.
pub fn verify_unencrypted(&self, expected_chunk_size: usize) -> Result<(), Error> {
let magic = &self.raw_data[0..8];
let magic = self.magic();
let verify_raw_data = |data: &[u8]| {
if expected_chunk_size != data.len() {
@ -204,10 +215,10 @@ impl DataChunk {
Ok(())
};
if magic == COMPRESSED_CHUNK_MAGIC_1_0 {
if magic == &COMPRESSED_CHUNK_MAGIC_1_0 {
let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
verify_raw_data(&data)?;
} else if magic == UNCOMPRESSED_CHUNK_MAGIC_1_0 {
} else if magic == &UNCOMPRESSED_CHUNK_MAGIC_1_0 {
verify_raw_data(&self.raw_data[12..])?;
}
@ -288,7 +299,12 @@ impl <'a, 'b> DataChunkBuilder<'a, 'b> {
self.compute_digest();
}
let chunk = DataChunk::new(self.orig_data, self.config, self.digest, self.compress)?;
let chunk = DataChunk::encode(
self.orig_data,
self.config,
self.digest,
self.compress,
)?;
Ok(chunk)
}

57
src/backup/file_formats.rs Normal file
View File

@ -0,0 +1,57 @@
// WARNING: PLEASE DO NOT MODIFY THOSE MAGIC VALUES
// openssl::sha::sha256(b"Proxmox Backup uncompressed chunk v1.0")[0..8]
pub static UNCOMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [79, 127, 200, 4, 121, 74, 135, 239];
// openssl::sha::sha256(b"Proxmox Backup encrypted chunk v1.0")[0..8]
pub static ENCRYPTED_CHUNK_MAGIC_1_0: [u8; 8] = [8, 54, 114, 153, 70, 156, 26, 151];
// openssl::sha::sha256(b"Proxmox Backup zstd compressed chunk v1.0")[0..8]
pub static COMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [191, 237, 46, 195, 108, 17, 228, 235];
// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted chunk v1.0")[0..8]
pub static ENCR_COMPR_CHUNK_MAGIC_1_0: [u8; 8] = [9, 40, 53, 200, 37, 150, 90, 196];
// openssl::sha::sha256(b"Proxmox Backup uncompressed blob v1.0")[0..8]
pub static UNCOMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [66, 171, 56, 7, 190, 131, 112, 161];
//openssl::sha::sha256(b"Proxmox Backup zstd compressed blob v1.0")[0..8]
pub static COMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [49, 185, 88, 66, 111, 182, 163, 127];
// openssl::sha::sha256(b"Proxmox Backup encrypted blob v1.0")[0..8]
pub static ENCRYPTED_BLOB_MAGIC_1_0: [u8; 8] = [123, 103, 133, 190, 34, 45, 76, 240];
// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted blob v1.0")[0..8]
pub static ENCR_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [230, 89, 27, 191, 11, 191, 216, 11];
// openssl::sha::sha256(b"Proxmox Backup fixed sized chunk index v1.0")[0..8]
pub static FIXED_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [47, 127, 65, 237, 145, 253, 15, 205];
// openssl::sha::sha256(b"Proxmox Backup dynamic sized chunk index v1.0")[0..8]
pub static DYNAMIC_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [28, 145, 78, 165, 25, 186, 179, 205];
#[repr(C,packed)]
pub struct DataBlobHeader {
pub magic: [u8; 8],
pub crc: [u8; 4],
}
#[repr(C,packed)]
pub struct EncryptedDataBlobHeader {
pub head: DataBlobHeader,
pub iv: [u8; 16],
pub tag: [u8; 16],
}
#[repr(C,packed)]
pub struct DataChunkHeader {
pub magic: [u8; 8],
pub crc: [u8; 4],
}
#[repr(C,packed)]
pub struct EncryptedDataChunkHeader {
pub head: DataChunkHeader,
pub iv: [u8; 16],
pub tag: [u8; 16],
}