From 3d5c11e5d2b6d7aa0ce6c0a8484425c1f2b81e99 Mon Sep 17 00:00:00 2001
From: Dietmar Maurer <dietmar@proxmox.com>
Date: Tue, 18 Dec 2018 11:06:03 +0100
Subject: [PATCH] start impl garbage collection

---
 src/backup/chunk_store.rs | 24 ++++++++++++++++----
 src/backup/datastore.rs   | 47 ++++++++++++++++++++++++++++++++++++++-
 src/backup/image_index.rs | 18 +++++++++++++++
 src/bin/backup-client.rs  |  2 ++
 4 files changed, 86 insertions(+), 5 deletions(-)

diff --git a/src/backup/chunk_store.rs b/src/backup/chunk_store.rs
index 009ee2cb..18a7285d 100644
--- a/src/backup/chunk_store.rs
+++ b/src/backup/chunk_store.rs
@@ -20,11 +20,11 @@ pub struct ChunkStore {
 
 const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef";
 
-pub fn u256_to_hex(digest: &[u8; 32]) -> String {
+pub fn u256_to_hex(digest: &[u8]) -> String {
 
-    let mut buf = Vec::<u8>::with_capacity(64);
+    let mut buf = Vec::<u8>::with_capacity(digest.len()*2);
 
-    for i in 0..32 {
+    for i in 0..digest.len() {
         buf.push(HEX_CHARS[(digest[i] >> 4) as usize]);
         buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]);
     }
@@ -32,7 +32,7 @@ pub fn u256_to_hex(digest: &[u8; 32]) -> String {
     unsafe { String::from_utf8_unchecked(buf) }
 }
 
-fn u256_to_prefix(digest: &[u8; 32]) -> PathBuf {
+fn u256_to_prefix(digest: &[u8]) -> PathBuf {
 
     let mut buf = Vec::<u8>::with_capacity(3+1+2+1);
 
@@ -151,6 +151,18 @@ impl ChunkStore {
         })
     }
 
+    pub fn touch_chunk(&mut self, digest:&[u8]) ->  Result<(), Error> {
+
+        let mut chunk_path = self.chunk_dir.clone();
+        let prefix = u256_to_prefix(&digest);
+        chunk_path.push(&prefix);
+        let digest_str = u256_to_hex(&digest);
+        chunk_path.push(&digest_str);
+
+        std::fs::metadata(&chunk_path)?;
+        Ok(())
+    }
+
     pub fn insert_chunk(&mut self, chunk: &[u8]) -> Result<(bool, [u8; 32]), Error> {
 
         self.hasher.reset();
@@ -205,6 +217,10 @@ impl ChunkStore {
         full_path
     }
 
+    pub fn base_path(&self) -> PathBuf {
+        self.base.clone()
+    }
+
 }
 
 
diff --git a/src/backup/datastore.rs b/src/backup/datastore.rs
index 3c6c5b7f..6767f7be 100644
--- a/src/backup/datastore.rs
+++ b/src/backup/datastore.rs
@@ -1,6 +1,6 @@
 use failure::*;
 
-use std::path::Path;
+use std::path::{PathBuf, Path};
 
 use crate::config::datastore;
 use super::chunk_store::*;
@@ -40,4 +40,49 @@ impl DataStore {
 
         Ok(index)
     }
+
+    pub fn list_images(&self) -> Result<Vec<PathBuf>, Error> {
+        let base = self.chunk_store.base_path();
+
+        let mut list = vec![];
+
+        for entry in std::fs::read_dir(base)? {
+            let entry = entry?;
+            if entry.file_type()?.is_file() {
+                let path = entry.path();
+                if let Some(ext) = path.extension() {
+                    if ext == "idx" {
+                        list.push(path);
+                    }
+                }
+            }
+        }
+
+        Ok(list)
+    }
+
+    fn sweep_used_chunks(&mut self) -> Result<(), Error> {
+
+        Ok(())
+    }
+
+    fn mark_used_chunks(&mut self) -> Result<(), Error> {
+
+        let image_list = self.list_images()?;
+
+        for path in image_list {
+            let mut index = self.open_image_reader(path)?;
+            index.mark_used_chunks();
+        }
+
+        Ok(())
+   }
+
+    pub fn garbage_collection(&mut self) -> Result<(), Error> {
+
+        self.mark_used_chunks()?;
+        self.sweep_used_chunks()?;
+
+        Ok(())
+    }
 }
diff --git a/src/backup/image_index.rs b/src/backup/image_index.rs
index 5194406d..5b9f6eba 100644
--- a/src/backup/image_index.rs
+++ b/src/backup/image_index.rs
@@ -100,6 +100,24 @@ impl <'a> ImageIndexReader<'a> {
         Ok(())
     }
 
+    pub fn mark_used_chunks(&mut self) -> Result<(), Error> {
+
+        if self.index == std::ptr::null_mut() { bail!("detected closed index file."); }
+
+        let index_count = (self.size + self.chunk_size - 1)/self.chunk_size;
+
+        for pos in 0..index_count {
+
+            let digest = unsafe { std::slice::from_raw_parts_mut(self.index.add(pos*32), 32) };
+            if let Err(err) = self.store.touch_chunk(digest) {
+                bail!("unable to access chunk {}, required by {:?} - {}",
+                      u256_to_hex(digest), self.filename, err);
+            }
+        }
+
+        Ok(())
+    }
+
     pub fn print_info(&self) {
         println!("Filename: {:?}", self.filename);
         println!("Size: {}", self.size);
diff --git a/src/bin/backup-client.rs b/src/bin/backup-client.rs
index e080f2ec..e1536ee4 100644
--- a/src/bin/backup-client.rs
+++ b/src/bin/backup-client.rs
@@ -46,6 +46,8 @@ fn backup_file(param: Value, _info: &ApiMethod) -> Result<Value, Error> {
         index.close()?; // commit changes
     }
 
+    datastore.garbage_collection()?;
+
     let idx = datastore.open_image_reader(target)?;
     idx.print_info();