diff --git a/src/backup/chunk_store.rs b/src/backup/chunk_store.rs index 009ee2cb..18a7285d 100644 --- a/src/backup/chunk_store.rs +++ b/src/backup/chunk_store.rs @@ -20,11 +20,11 @@ pub struct ChunkStore { const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef"; -pub fn u256_to_hex(digest: &[u8; 32]) -> String { +pub fn u256_to_hex(digest: &[u8]) -> String { - let mut buf = Vec::::with_capacity(64); + let mut buf = Vec::::with_capacity(digest.len()*2); - for i in 0..32 { + for i in 0..digest.len() { buf.push(HEX_CHARS[(digest[i] >> 4) as usize]); buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]); } @@ -32,7 +32,7 @@ pub fn u256_to_hex(digest: &[u8; 32]) -> String { unsafe { String::from_utf8_unchecked(buf) } } -fn u256_to_prefix(digest: &[u8; 32]) -> PathBuf { +fn u256_to_prefix(digest: &[u8]) -> PathBuf { let mut buf = Vec::::with_capacity(3+1+2+1); @@ -151,6 +151,18 @@ impl ChunkStore { }) } + pub fn touch_chunk(&mut self, digest:&[u8]) -> Result<(), Error> { + + let mut chunk_path = self.chunk_dir.clone(); + let prefix = u256_to_prefix(&digest); + chunk_path.push(&prefix); + let digest_str = u256_to_hex(&digest); + chunk_path.push(&digest_str); + + std::fs::metadata(&chunk_path)?; + Ok(()) + } + pub fn insert_chunk(&mut self, chunk: &[u8]) -> Result<(bool, [u8; 32]), Error> { self.hasher.reset(); @@ -205,6 +217,10 @@ impl ChunkStore { full_path } + pub fn base_path(&self) -> PathBuf { + self.base.clone() + } + } diff --git a/src/backup/datastore.rs b/src/backup/datastore.rs index 3c6c5b7f..6767f7be 100644 --- a/src/backup/datastore.rs +++ b/src/backup/datastore.rs @@ -1,6 +1,6 @@ use failure::*; -use std::path::Path; +use std::path::{PathBuf, Path}; use crate::config::datastore; use super::chunk_store::*; @@ -40,4 +40,49 @@ impl DataStore { Ok(index) } + + pub fn list_images(&self) -> Result, Error> { + let base = self.chunk_store.base_path(); + + let mut list = vec![]; + + for entry in std::fs::read_dir(base)? { + let entry = entry?; + if entry.file_type()?.is_file() { + let path = entry.path(); + if let Some(ext) = path.extension() { + if ext == "idx" { + list.push(path); + } + } + } + } + + Ok(list) + } + + fn sweep_used_chunks(&mut self) -> Result<(), Error> { + + Ok(()) + } + + fn mark_used_chunks(&mut self) -> Result<(), Error> { + + let image_list = self.list_images()?; + + for path in image_list { + let mut index = self.open_image_reader(path)?; + index.mark_used_chunks(); + } + + Ok(()) + } + + pub fn garbage_collection(&mut self) -> Result<(), Error> { + + self.mark_used_chunks()?; + self.sweep_used_chunks()?; + + Ok(()) + } } diff --git a/src/backup/image_index.rs b/src/backup/image_index.rs index 5194406d..5b9f6eba 100644 --- a/src/backup/image_index.rs +++ b/src/backup/image_index.rs @@ -100,6 +100,24 @@ impl <'a> ImageIndexReader<'a> { Ok(()) } + pub fn mark_used_chunks(&mut self) -> Result<(), Error> { + + if self.index == std::ptr::null_mut() { bail!("detected closed index file."); } + + let index_count = (self.size + self.chunk_size - 1)/self.chunk_size; + + for pos in 0..index_count { + + let digest = unsafe { std::slice::from_raw_parts_mut(self.index.add(pos*32), 32) }; + if let Err(err) = self.store.touch_chunk(digest) { + bail!("unable to access chunk {}, required by {:?} - {}", + u256_to_hex(digest), self.filename, err); + } + } + + Ok(()) + } + pub fn print_info(&self) { println!("Filename: {:?}", self.filename); println!("Size: {}", self.size); diff --git a/src/bin/backup-client.rs b/src/bin/backup-client.rs index e080f2ec..e1536ee4 100644 --- a/src/bin/backup-client.rs +++ b/src/bin/backup-client.rs @@ -46,6 +46,8 @@ fn backup_file(param: Value, _info: &ApiMethod) -> Result { index.close()?; // commit changes } + datastore.garbage_collection()?; + let idx = datastore.open_image_reader(target)?; idx.print_info();