From 606ce64bb01b4b4c16a50ebc229bc0eb6b6b78b7 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 15 Dec 2018 14:51:05 +0100 Subject: [PATCH] image_index.rs:write idx of chunks (use mmap) --- src/backup/chunk_store.rs | 11 ++++- src/backup/image_index.rs | 84 +++++++++++++++++++++++++++++++++++++++ src/bin/backup-client.rs | 11 ++++- src/lib.rs | 1 + src/tools.rs | 4 +- 5 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 src/backup/image_index.rs diff --git a/src/backup/chunk_store.rs b/src/backup/chunk_store.rs index 53bec858..7f76fe11 100644 --- a/src/backup/chunk_store.rs +++ b/src/backup/chunk_store.rs @@ -20,7 +20,7 @@ pub struct ChunkStore { const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef"; -fn u256_to_hex(digest: &[u8; 32]) -> String { +pub fn u256_to_hex(digest: &[u8; 32]) -> String { let mut buf = Vec::::with_capacity(64); @@ -158,7 +158,7 @@ impl ChunkStore { let mut digest = [0u8; 32]; self.hasher.result(&mut digest); - println!("DIGEST {}", u256_to_hex(&digest)); + //println!("DIGEST {}", u256_to_hex(&digest)); let mut chunk_path = self.base.clone(); let prefix = u256_to_prefix(&digest); @@ -198,6 +198,13 @@ impl ChunkStore { Ok((false, digest)) } + pub fn relative_path(&self, path: &Path) -> PathBuf { + + let mut full_path = self.base.clone(); + full_path.push(path); + full_path + } + } diff --git a/src/backup/image_index.rs b/src/backup/image_index.rs new file mode 100644 index 00000000..ffd2f86a --- /dev/null +++ b/src/backup/image_index.rs @@ -0,0 +1,84 @@ +use failure::*; + +use super::chunk_store::*; + +use std::path::{Path, PathBuf}; +use std::os::unix::io::AsRawFd; + + +// split image into fixed size chunks + +pub struct ImageIndex<'a> { + store: &'a mut ChunkStore, + chunk_size: usize, + size: usize, + index: *mut u8, +} + +impl <'a> ImageIndex<'a> { + + pub fn create(store: &'a mut ChunkStore, path: &Path, size: usize) -> Result { + + let full_path = store.relative_path(path); + println!("FULLPATH: {:?} {}", full_path, size); + + let mut file = std::fs::OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&full_path)?; + + let chunk_size = 64*1024; + let index_size = ((size + chunk_size - 1)/chunk_size)*32; + nix::unistd::ftruncate(file.as_raw_fd(), index_size as i64)?; + println!("SIZES: {}", index_size); + + let data = unsafe { nix::sys::mman::mmap( + std::ptr::null_mut(), + index_size, + nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE, + nix::sys::mman::MapFlags::MAP_SHARED, + file.as_raw_fd(), + 0) }? as *mut u8; + + Ok(Self { + store, + chunk_size, + size, + index: data, + }) + } + + // Note: We want to add data out of order, so do not assume and order here. + pub fn add_chunk(&mut self, pos: usize, chunk: &[u8]) -> Result<(), Error> { + + let end = pos + chunk.len(); + + if end > self.size { + bail!("write chunk data exceeds size ({} >= {})", end, self.size); + } + + // last chunk can be smaller + if ((end != self.size) && (chunk.len() != self.chunk_size)) || + (chunk.len() > self.chunk_size) || (chunk.len() == 0) { + bail!("got chunk with wrong length ({} != {}", chunk.len(), self.chunk_size); + } + + if pos >= self.size { bail!("add chunk after end ({} >= {})", pos, self.size); } + + if pos & (self.chunk_size-1) != 0 { bail!("add unaligned chunk (pos = {})", pos); } + + + let (is_duplicate, digest) = self.store.insert_chunk(chunk)?; + + println!("ADD CHUNK {} {} {} {}", pos, chunk.len(), is_duplicate, u256_to_hex(&digest)); + + let index_pos = (pos/self.chunk_size)*32; + unsafe { + let dst = self.index.add(index_pos); + dst.copy_from_nonoverlapping(digest.as_ptr(), 32); + } + + Ok(()) + } +} diff --git a/src/bin/backup-client.rs b/src/bin/backup-client.rs index 6842b989..8291331d 100644 --- a/src/bin/backup-client.rs +++ b/src/bin/backup-client.rs @@ -1,12 +1,14 @@ extern crate apitest; use failure::*; +use std::os::unix::io::AsRawFd; use apitest::tools; use apitest::cli::command::*; use apitest::api::schema::*; use apitest::api::router::*; use apitest::backup::chunk_store::*; +use apitest::backup::image_index::*; use serde_json::{Value}; use apitest::config::datastore; @@ -27,14 +29,19 @@ fn backup_file(param: Value, _info: &ApiMethod) -> Result { let path = store_config["path"].as_str().unwrap(); - let _store = ChunkStore::open(path)?; + let mut chunk_store = ChunkStore::open(path)?; println!("Backup file '{}' to '{}'", filename, store); let file = std::fs::File::open(filename)?; + let stat = nix::sys::stat::fstat(file.as_raw_fd())?; + if stat.st_size <= 0 { bail!("got strange file size '{}'", stat.st_size); } + let size = stat.st_size as usize; + + let mut index = ImageIndex::create(&mut chunk_store, "test1.idx".as_ref(), size)?; tools::file_chunker(file, 64*1024, |pos, chunk| { - println!("CHUNK {} {}", pos, chunk.len()); + index.add_chunk(pos, chunk)?; Ok(true) })?; diff --git a/src/lib.rs b/src/lib.rs index 808c988e..011232fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,6 +35,7 @@ pub mod section_config; pub mod backup { pub mod chunk_store; + pub mod image_index; } pub mod config { diff --git a/src/tools.rs b/src/tools.rs index 8cacabe0..9eb54ec7 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -58,9 +58,9 @@ pub fn file_set_contents>( pub fn file_chunker( mut file: R, chunk_size: usize, - chunk_cb: C + mut chunk_cb: C ) -> Result<(), Error> - where C: Fn(usize, &[u8]) -> Result, + where C: FnMut(usize, &[u8]) -> Result, R: Read, {