From 96482891ae28727f717d60dd641c902001bbc667 Mon Sep 17 00:00:00 2001
From: Dietmar Maurer <dietmar@proxmox.com>
Date: Thu, 30 May 2019 09:20:32 +0200
Subject: [PATCH] src/api2/admin/datastore/backup/environment.rs: log upload
 statistics

---
 .../admin/datastore/backup/environment.rs     | 65 +++++++++++++++++--
 1 file changed, 61 insertions(+), 4 deletions(-)

diff --git a/src/api2/admin/datastore/backup/environment.rs b/src/api2/admin/datastore/backup/environment.rs
index 0ebf7455..006a2dce 100644
--- a/src/api2/admin/datastore/backup/environment.rs
+++ b/src/api2/admin/datastore/backup/environment.rs
@@ -10,12 +10,30 @@ use crate::backup::*;
 use crate::server::formatter::*;
 use hyper::{Body, Response};
 
+struct UploadStatistic {
+    count: u64,
+    size: u64,
+    compressed_size: u64,
+    duplicates: u64,
+}
+
+impl UploadStatistic {
+    fn new() -> Self {
+        Self {
+            count: 0,
+            size: 0,
+            compressed_size: 0,
+            duplicates: 0,
+        }
+    }
+}
 
 struct DynamicWriterState {
     name: String,
     index: DynamicIndexWriter,
     offset: u64,
     chunk_count: u64,
+    upload_stat: UploadStatistic,
 }
 
 struct FixedWriterState {
@@ -24,6 +42,7 @@ struct FixedWriterState {
     size: usize,
     chunk_size: u32,
     chunk_count: u64,
+    upload_stat: UploadStatistic,
 }
 
 struct SharedBackupState {
@@ -100,8 +119,10 @@ impl BackupEnvironment {
         }
     }
 
-    // Register a Chunk with associated length. A client may only use registered
-    // chunks (we do not trust clients that far ...)
+    /// Register a Chunk with associated length.
+    ///
+    /// We do not fully trust clients, so a client may only use registered
+    /// chunks. Please use this method to register chunks from previous backups.
     pub fn register_chunk(&self, digest: [u8; 32], length: u32) -> Result<(), Error> {
         let mut state = self.state.lock().unwrap();
 
@@ -112,6 +133,10 @@ impl BackupEnvironment {
         Ok(())
     }
 
+    /// Register fixed length chunks after upload.
+    ///
+    /// Like `register_chunk()`, but additionally record statistics for
+    /// the fixed index writer.
     pub fn register_fixed_chunk(
         &self,
         wid: usize,
@@ -133,11 +158,22 @@ impl BackupEnvironment {
             bail!("fixed writer '{}' - got unexpected chunk size ({} != {}", data.name, size, data.chunk_size);
         }
 
+        // record statistics
+        data.upload_stat.count += 1;
+        data.upload_stat.size += size as u64;
+        data.upload_stat.compressed_size += compressed_size as u64;
+        if is_duplicate { data.upload_stat.duplicates += 1; }
+
+        // register chunk
         state.known_chunks.insert(digest, size);
 
         Ok(())
     }
 
+    /// Register dynamic length chunks after upload.
+    ///
+    /// Like `register_chunk()`, but additionally record statistics for
+    /// the dynamic index writer.
     pub fn register_dynamic_chunk(
         &self,
         wid: usize,
@@ -155,6 +191,13 @@ impl BackupEnvironment {
             None => bail!("dynamic writer '{}' not registered", wid),
         };
 
+        // record statistics
+        data.upload_stat.count += 1;
+        data.upload_stat.size += size as u64;
+        data.upload_stat.compressed_size += compressed_size as u64;
+        if is_duplicate { data.upload_stat.duplicates += 1; }
+
+        // register chunk
         state.known_chunks.insert(digest, size);
 
         Ok(())
@@ -178,7 +221,7 @@ impl BackupEnvironment {
         let uid = state.next_uid();
 
         state.dynamic_writers.insert(uid, DynamicWriterState {
-            index, name, offset: 0, chunk_count: 0,
+            index, name, offset: 0, chunk_count: 0, upload_stat: UploadStatistic::new(),
         });
 
         Ok(uid)
@@ -193,7 +236,7 @@ impl BackupEnvironment {
         let uid = state.next_uid();
 
         state.fixed_writers.insert(uid, FixedWriterState {
-            index, name, chunk_count: 0, size, chunk_size,
+            index, name, chunk_count: 0, size, chunk_size, upload_stat: UploadStatistic::new(),
         });
 
         Ok(uid)
@@ -247,6 +290,16 @@ impl BackupEnvironment {
         Ok(())
     }
 
+    fn log_upload_stat(&self, archive_name: &str, size: u64, chunk_count: u64, upload_stat: &UploadStatistic) {
+        self.log(format!("Upload statistics for '{}'", archive_name));
+        self.log(format!("Size: {}", size));
+        self.log(format!("Chunk count: {}", chunk_count));
+        self.log(format!("Upload size: {} ({}%)", upload_stat.size, (upload_stat.size*100)/size));
+        if upload_stat.size > 0 {
+            self.log(format!("Compression: {}%",  (upload_stat.compressed_size*100)/upload_stat.size));
+        }
+    }
+
     /// Close dynamic writer
     pub fn dynamic_writer_close(&self, wid: usize, chunk_count: u64, size: u64) -> Result<(), Error> {
         let mut state = self.state.lock().unwrap();
@@ -268,6 +321,8 @@ impl BackupEnvironment {
 
         data.index.close()?;
 
+        self.log_upload_stat(&data.name, size, chunk_count, &data.upload_stat);
+
         state.file_counter += 1;
 
         Ok(())
@@ -300,6 +355,8 @@ impl BackupEnvironment {
 
         data.index.close()?;
 
+        self.log_upload_stat(&data.name, size, chunk_count, &data.upload_stat);
+
         state.file_counter += 1;
 
         Ok(())