tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

backup: Add support for client side encryption

Browse Source 
first try ...
This commit is contained in:
Dietmar Maurer
2019-06-13 11:47:23 +02:00
parent 51929e4532
commit f98ac774ee
7 changed files with 187 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/backup/chunk_store.rs
View File

@ -1,14 +1,13 @@
use failure::*;
use std::path::{Path, PathBuf};
use std::io::{Read, Write};
use std::io::Write;
use std::sync::{Arc, Mutex};
use std::os::unix::io::AsRawFd;
use serde_derive::Serialize;
use openssl::sha;
use crate::tools;
use super::DataChunk;
#[derive(Clone, Serialize)]
pub struct GarbageCollectionStatus {
@ -173,21 +172,19 @@ impl ChunkStore {
Ok(())
}
pub fn read_chunk(&self, digest:&[u8], buffer: &mut Vec<u8>) -> Result<(), Error> {
pub fn read_chunk(&self, digest:&[u8; 32]) -> Result<DataChunk, Error> {
let mut chunk_path = self.chunk_dir.clone();
let prefix = digest_to_prefix(&digest);
let prefix = digest_to_prefix(digest);
chunk_path.push(&prefix);
let digest_str = tools::digest_to_hex(&digest);
let digest_str = tools::digest_to_hex(digest);
chunk_path.push(&digest_str);
buffer.clear();
let f = std::fs::File::open(&chunk_path)?;
let mut decoder = zstd::stream::Decoder::new(f)?;
let mut file = std::fs::File::open(&chunk_path)
.map_err(|err| format_err!(
"store '{}', unable to read chunk '{}' - {}", self.name, digest_str, err))?;
decoder.read_to_end(buffer)?;
Ok(())
DataChunk::load(&mut file, *digest)
}
pub fn get_chunk_iterator(
@ -320,21 +317,14 @@ impl ChunkStore {
Ok(())
}
pub fn insert_chunk(&self, chunk: &[u8]) -> Result<(bool, [u8; 32], u64), Error> {
// fixme: use Sha512/256 when available
let digest = sha::sha256(chunk);
let (new, csize) = self.insert_chunk_noverify(&digest, chunk)?;
Ok((new, digest, csize))
}
pub fn insert_chunk_noverify(
pub fn insert_chunk(
&self,
digest: &[u8; 32],
chunk: &[u8],
chunk: &DataChunk,
) -> Result<(bool, u64), Error> {
//println!("DIGEST {}", tools::digest_to_hex(&digest));
let digest = chunk.digest();
//println!("DIGEST {}", tools::digest_to_hex(digest));
let mut chunk_path = self.chunk_dir.clone();
let prefix = digest_to_prefix(digest);
@ -355,12 +345,12 @@ impl ChunkStore {
let mut tmp_path = chunk_path.clone();
tmp_path.set_extension("tmp");
let f = std::fs::File::create(&tmp_path)?;
let mut file = std::fs::File::create(&tmp_path)?;
let mut encoder = zstd::stream::Encoder::new(f, 1)?;
let raw_data = chunk.raw_data();
let encoded_size = raw_data.len() as u64;
encoder.write_all(chunk)?;
let f = encoder.finish()?;
file.write_all(raw_data)?;
if let Err(err) = std::fs::rename(&tmp_path, &chunk_path) {
if let Err(_) = std::fs::remove_file(&tmp_path) { /* ignore */ }
@ -372,15 +362,9 @@ impl ChunkStore {
);
}
// fixme: is there a better way to get the compressed size?
let stat = nix::sys::stat::fstat(f.as_raw_fd())?;
let compressed_size = stat.st_size as u64;
//println!("PATH {:?}", chunk_path);
drop(lock);
Ok((false, compressed_size))
Ok((false, encoded_size))
}
pub fn relative_path(&self, path: &Path) -> PathBuf {
@ -416,10 +400,13 @@ fn test_chunk_store1() {
assert!(chunk_store.is_err());
let chunk_store = ChunkStore::create("test", &path).unwrap();
let (exists, _, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap();
let chunk = super::DataChunkBuilder::new(&[0u8, 1u8]).build().unwrap();
let (exists, _) = chunk_store.insert_chunk(&chunk).unwrap();
assert!(!exists);
let (exists, _, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap();
let (exists, _) = chunk_store.insert_chunk(&chunk).unwrap();
assert!(exists);

14
src/backup/datastore.rs
View File

@ -13,6 +13,7 @@ use super::fixed_index::*;
use super::dynamic_index::*;
use super::index::*;
use super::backup_info::*;
use super::DataChunk;
use crate::server::WorkerTask;
lazy_static!{
@ -256,15 +257,10 @@ impl DataStore {
Ok(())
}
pub fn insert_chunk(&self, chunk: &[u8]) -> Result<(bool, [u8; 32], u64), Error> {
pub fn insert_chunk(
&self,
chunk: &DataChunk,
) -> Result<(bool, u64), Error> {
self.chunk_store.insert_chunk(chunk)
}
pub fn insert_chunk_noverify(
&self,
digest: &[u8; 32],
chunk: &[u8],
) -> Result<(bool, u64), Error> {
self.chunk_store.insert_chunk_noverify(digest, chunk)
}
}

41
src/backup/dynamic_index.rs
View File

@ -1,4 +1,5 @@
use failure::*;
use std::convert::TryInto;
use crate::tools;
use super::IndexFile;
@ -17,6 +18,8 @@ use uuid::Uuid;
use crate::tools::io::ops::*;
use crate::tools::vec;
use super::{DataChunk, DataChunkBuilder};
/// Header format definition for dynamic index files (`.dixd`)
#[repr(C)]
pub struct DynamicIndexHeader {
@ -158,11 +161,12 @@ impl DynamicIndexReader {
}
#[inline]
fn chunk_digest(&self, pos: usize) -> &[u8] {
fn chunk_digest(&self, pos: usize) -> &[u8; 32] {
if pos >= self.index_entries {
panic!("chunk index out of range");
}
unsafe { std::slice::from_raw_parts(self.index.add(pos*40+8), 32) }
let slice = unsafe { std::slice::from_raw_parts(self.index.add(pos*40+8), 32) };
slice.try_into().unwrap()
}
pub fn mark_used_chunks(&self, _status: &mut GarbageCollectionStatus) -> Result<(), Error> {
@ -182,15 +186,14 @@ impl DynamicIndexReader {
pub fn dump_pxar(&self, mut writer: Box<dyn Write>) -> Result<(), Error> {
let mut buffer = Vec::with_capacity(1024*1024);
for pos in 0..self.index_entries {
let _end = self.chunk_end(pos);
let digest = self.chunk_digest(pos);
//println!("Dump {:08x}", end );
self.store.read_chunk(digest, &mut buffer)?;
writer.write_all(&buffer)?;
let chunk = self.store.read_chunk(digest)?;
// fimxe: handle encrypted chunks
let data = chunk.decode(None)?;
writer.write_all(&data)?;
}
Ok(())
@ -270,7 +273,14 @@ impl BufferedDynamicReader {
let index = &self.index;
let end = index.chunk_end(idx);
let digest = index.chunk_digest(idx);
index.store.read_chunk(digest, &mut self.read_buffer)?;
let chunk = index.store.read_chunk(digest)?;
// fimxe: handle encrypted chunks
// fixme: avoid copy
let data = chunk.decode(None)?;
self.read_buffer.clear();
self.read_buffer.extend_from_slice(&data);
self.buffered_chunk_idx = idx;
self.buffered_chunk_start = end - (self.read_buffer.len() as u64);
@ -433,7 +443,8 @@ impl DynamicIndexWriter {
})
}
pub fn insert_chunk(&self, chunk: &[u8]) -> Result<(bool, [u8; 32], u64), Error> {
// fixme: use add_chunk instead?
pub fn insert_chunk(&self, chunk: &DataChunk) -> Result<(bool, u64), Error> {
self.store.insert_chunk(chunk)
}
@ -531,8 +542,14 @@ impl DynamicChunkWriter {
self.last_chunk = self.chunk_offset;
match self.index.insert_chunk(&self.chunk_buffer) {
Ok((is_duplicate, digest, compressed_size)) => {
let chunk = DataChunkBuilder::new(&self.chunk_buffer)
.compress(true)
.build()?;
let digest = chunk.digest();
match self.index.insert_chunk(&chunk) {
Ok((is_duplicate, compressed_size)) => {
self.stat.compressed_size += compressed_size;
if is_duplicate {
@ -542,7 +559,7 @@ impl DynamicChunkWriter {
}
println!("ADD CHUNK {:016x} {} {}% {} {}", self.chunk_offset, chunk_size,
(compressed_size*100)/(chunk_size as u64), is_duplicate, tools::digest_to_hex(&digest));
(compressed_size*100)/(chunk_size as u64), is_duplicate, tools::digest_to_hex(digest));
self.index.add_chunk(self.chunk_offset as u64, &digest)?;
self.chunk_buffer.truncate(0);
return Ok(());

32
src/backup/fixed_index.rs
View File

@ -12,6 +12,7 @@ use std::path::{Path, PathBuf};
use std::os::unix::io::AsRawFd;
use uuid::Uuid;
use chrono::{Local, TimeZone};
use super::ChunkInfo;
/// Header format definition for fixed index files (`.fidx`)
#[repr(C)]
@ -307,29 +308,42 @@ impl FixedIndexWriter {
}
// Note: We want to add data out of order, so do not assume any order here.
pub fn add_chunk(&mut self, pos: usize, chunk: &[u8], stat: &mut ChunkStat) -> Result<(), Error> {
pub fn add_chunk(&mut self, chunk_info: &ChunkInfo, stat: &mut ChunkStat) -> Result<(), Error> {
let end = pos + chunk.len();
let chunk_len = chunk_info.chunk_len as usize;
let end = chunk_info.offset as usize;
if end < chunk_len {
bail!("got chunk with small offset ({} < {}", end, chunk_len);
}
let pos = end - chunk_len;
if end > self.size {
bail!("write chunk data exceeds size ({} >= {})", end, self.size);
}
// last chunk can be smaller
if ((end != self.size) && (chunk.len() != self.chunk_size)) ||
(chunk.len() > self.chunk_size) || (chunk.len() == 0) {
bail!("got chunk with wrong length ({} != {}", chunk.len(), self.chunk_size);
if ((end != self.size) && (chunk_len != self.chunk_size)) ||
(chunk_len > self.chunk_size) || (chunk_len == 0) {
bail!("got chunk with wrong length ({} != {}", chunk_len, self.chunk_size);
}
if pos & (self.chunk_size-1) != 0 { bail!("add unaligned chunk (pos = {})", pos); }
let (is_duplicate, digest, compressed_size) = self.store.insert_chunk(chunk)?;
if (end as u64) != chunk_info.offset {
bail!("got chunk with wrong offset ({} != {}", end, chunk_info.offset);
}
let (is_duplicate, compressed_size) = self.store.insert_chunk(&chunk_info.chunk)?;
stat.chunk_count += 1;
stat.compressed_size += compressed_size;
println!("ADD CHUNK {} {} {}% {} {}", pos, chunk.len(),
(compressed_size*100)/(chunk.len() as u64), is_duplicate, tools::digest_to_hex(&digest));
let digest = chunk_info.chunk.digest();
println!("ADD CHUNK {} {} {}% {} {}", pos, chunk_len,
(compressed_size*100)/(chunk_len as u64), is_duplicate, tools::digest_to_hex(digest));
if is_duplicate {
stat.duplicate_chunks += 1;
@ -337,7 +351,7 @@ impl FixedIndexWriter {
stat.disk_size += compressed_size;
}
self.add_digest(pos / self.chunk_size, &digest)
self.add_digest(pos / self.chunk_size, digest)
}
pub fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> {