From 64e53b2835bd7ba49e4bb665512f757a85bb2992 Mon Sep 17 00:00:00 2001
From: Dietmar Maurer <dietmar@proxmox.com>
Date: Sat, 22 Dec 2018 16:58:16 +0100
Subject: [PATCH] gather usage statistics during garbage collection

---
 src/backup/chunk_store.rs | 28 +++++++++++++++++++++++++---
 src/backup/datastore.rs   | 31 ++++++++++++++++++++++++-------
 src/backup/image_index.rs |  5 ++++-
 3 files changed, 53 insertions(+), 11 deletions(-)

diff --git a/src/backup/chunk_store.rs b/src/backup/chunk_store.rs
index 7acdb491..784d02b3 100644
--- a/src/backup/chunk_store.rs
+++ b/src/backup/chunk_store.rs
@@ -11,6 +11,24 @@ use std::os::unix::io::AsRawFd;
 
 use crate::tools;
 
+pub struct GarbageCollectionStatus {
+    pub used_bytes: usize,
+    pub used_chunks: usize,
+    pub disk_bytes: usize,
+    pub disk_chunks: usize,
+}
+
+impl Default for GarbageCollectionStatus {
+    fn default() -> Self {
+        GarbageCollectionStatus {
+            used_bytes: 0,
+            used_chunks: 0,
+            disk_bytes: 0,
+            disk_chunks: 0,
+        }
+    }
+}
+
 pub struct ChunkStore {
     name: String, // used for error reporting
     base: PathBuf,
@@ -140,7 +158,7 @@ impl ChunkStore {
         Ok(())
     }
 
-    fn sweep_old_files(&self, handle: &mut nix::dir::Dir) -> Result<(), Error> {
+    fn sweep_old_files(&self, handle: &mut nix::dir::Dir, status: &mut GarbageCollectionStatus) -> Result<(), Error> {
 
         let rawfd = handle.as_raw_fd();
 
@@ -168,13 +186,17 @@ impl ChunkStore {
                         let err = nix::Error::last();
                         bail!("unlink chunk {:?} failed on store '{}' - {}", filename, self.name, err);
                     }
+                } else {
+                    status.disk_chunks += 1;
+                    status.disk_bytes += stat.st_size as usize;
+
                 }
             }
         }
         Ok(())
     }
 
-    pub fn sweep_used_chunks(&self) -> Result<(), Error> {
+    pub fn sweep_used_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> {
 
         use nix::fcntl::OFlag;
         use nix::sys::stat::Mode;
@@ -219,7 +241,7 @@ impl ChunkStore {
                         "unable to open store '{}' dir {:?}/{:?}/{:?} - {}",
                         self.name, self.chunk_dir, l1name, l2name, err),
                 };
-                self.sweep_old_files(&mut l2_handle)?;
+                self.sweep_old_files(&mut l2_handle, status)?;
             }
         }
         Ok(())
diff --git a/src/backup/datastore.rs b/src/backup/datastore.rs
index ba092816..571e9396 100644
--- a/src/backup/datastore.rs
+++ b/src/backup/datastore.rs
@@ -1,6 +1,7 @@
 use failure::*;
 
 use std::path::{PathBuf, Path};
+use std::sync::Mutex;
 
 use crate::config::datastore;
 use super::chunk_store::*;
@@ -8,6 +9,7 @@ use super::image_index::*;
 
 pub struct DataStore {
     chunk_store: ChunkStore,
+    gc_mutex: Mutex<bool>,
 }
 
 impl DataStore {
@@ -24,6 +26,7 @@ impl DataStore {
 
         Ok(Self {
             chunk_store: chunk_store,
+            gc_mutex: Mutex::new(false),
         })
     }
 
@@ -51,7 +54,7 @@ impl DataStore {
             if entry.file_type()?.is_file() {
                 let path = entry.path();
                 if let Some(ext) = path.extension() {
-                    if ext == "idx" {
+                    if ext == "iidx" {
                         list.push(path);
                     }
                 }
@@ -61,13 +64,13 @@ impl DataStore {
         Ok(list)
     }
 
-    fn mark_used_chunks(&self) -> Result<(), Error> {
+    fn mark_used_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> {
 
         let image_list = self.list_images()?;
 
         for path in image_list {
             let index = self.open_image_reader(path)?;
-            index.mark_used_chunks()?;
+            index.mark_used_chunks(status)?;
         }
 
         Ok(())
@@ -75,12 +78,26 @@ impl DataStore {
 
     pub fn garbage_collection(&self) -> Result<(), Error> {
 
-        println!("Start GC phase1 (mark chunks)");
+        if let Ok(ref mut mutex) = self.gc_mutex.try_lock() {
 
-        self.mark_used_chunks()?;
+            let mut gc_status = GarbageCollectionStatus::default();
+            gc_status.used_bytes = 0;
 
-        println!("Start GC phase2 (sweep unused chunks)");
-        self.chunk_store.sweep_used_chunks()?;
+            println!("Start GC phase1 (mark chunks)");
+
+            self.mark_used_chunks(&mut gc_status)?;
+
+            println!("Start GC phase2 (sweep unused chunks)");
+            self.chunk_store.sweep_used_chunks(&mut gc_status)?;
+
+            println!("Used bytes: {}", gc_status.used_bytes);
+            println!("Used chunks: {}", gc_status.used_chunks);
+            println!("Disk bytes: {}", gc_status.disk_bytes);
+            println!("Disk chunks: {}", gc_status.disk_chunks);
+
+        } else {
+            println!("Start GC failed - (already running/locked)");
+        }
 
         Ok(())
     }
diff --git a/src/backup/image_index.rs b/src/backup/image_index.rs
index fb521a61..2372698e 100644
--- a/src/backup/image_index.rs
+++ b/src/backup/image_index.rs
@@ -100,12 +100,15 @@ impl <'a> ImageIndexReader<'a> {
         Ok(())
     }
 
-    pub fn mark_used_chunks(&self) -> Result<(), Error> {
+    pub fn mark_used_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> {
 
         if self.index == std::ptr::null_mut() { bail!("detected closed index file."); }
 
         let index_count = (self.size + self.chunk_size - 1)/self.chunk_size;
 
+        status.used_bytes += index_count * self.chunk_size;
+        status.used_chunks += index_count;
+
         for pos in 0..index_count {
 
             let digest = unsafe { std::slice::from_raw_parts_mut(self.index.add(pos*32), 32) };