move chunk_stat, read_chunk to pbs-datastore

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>

src/backup/cached_chunk_reader.rs

@@ -1,20 +1,22 @@
 //! An async and concurrency safe data reader backed by a local LRU cache.
 
-use anyhow::Error;
-use futures::future::Future;
-use futures::ready;
-use tokio::io::{AsyncRead, AsyncSeek, ReadBuf};
-
+use std::future::Future;
 use std::io::SeekFrom;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
 
-use super::{AsyncReadChunk, IndexFile};
-use crate::tools::async_lru_cache::{AsyncCacher, AsyncLruCache};
+use anyhow::Error;
+use futures::ready;
+use tokio::io::{AsyncRead, AsyncSeek, ReadBuf};
+
 use proxmox::io_format_err;
 use proxmox::sys::error::io_err_other;
 
+use pbs_datastore::read_chunk::AsyncReadChunk;
+use super::IndexFile;
+use crate::tools::async_lru_cache::{AsyncCacher, AsyncLruCache};
+
 struct AsyncChunkCacher<T> {
     reader: Arc<T>,
 }

src/backup/chunk_stat.rs

@@ -1,43 +0,0 @@
-pub struct ChunkStat {
-    pub size: u64,
-    pub compressed_size: u64,
-    pub disk_size: u64,
-
-    pub chunk_count: usize,
-    pub duplicate_chunks: usize,
-
-    start_time: std::time::SystemTime,
-}
-
-impl ChunkStat {
-
-    pub fn new(size: u64) -> Self {
-        ChunkStat {
-            size,
-            compressed_size: 0,
-            disk_size: 0,
-
-            chunk_count: 0,
-            duplicate_chunks: 0,
-
-            start_time: std::time::SystemTime::now(),
-        }
-    }
-}
-
-impl std::fmt::Debug for ChunkStat {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        let avg = ((self.size as f64)/(self.chunk_count as f64)) as usize;
-        let compression = (self.compressed_size*100)/(self.size as u64);
-        let rate = (self.disk_size*100)/(self.size as u64);
-
-        let elapsed = self.start_time.elapsed().unwrap();
-        let elapsed = (elapsed.as_secs() as f64) +
-            (elapsed.subsec_millis() as f64)/1000.0;
-
-        let write_speed = ((self.size as f64)/(1024.0*1024.0))/elapsed;
-
-        write!(f, "Size: {}, average chunk size: {}, compression rate: {}%, disk_size: {} ({}%), speed: {:.2} MB/s",
-               self.size, avg, compression, self.disk_size, rate, write_speed)
-    }
-}

src/backup/dynamic_index.rs

@@ -14,14 +14,13 @@ use proxmox::tools::uuid::Uuid;
 use proxmox::tools::mmap::Mmap;
 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
 
-use super::chunk_stat::ChunkStat;
-use super::chunk_store::ChunkStore;
-use super::index::ChunkReadInfo;
-use super::read_chunk::ReadChunk;
-use super::Chunker;
-use super::IndexFile;
-use super::{DataBlob, DataChunkBuilder};
-use crate::tools;
+use pbs_datastore::Chunker;
+use pbs_datastore::index::{IndexFile, ChunkReadInfo};
+use pbs_datastore::chunk_stat::ChunkStat;
+use pbs_datastore::data_blob::{DataBlob, DataChunkBuilder};
+use pbs_datastore::chunk_store::ChunkStore;
+use pbs_datastore::read_chunk::ReadChunk;
+use pbs_tools::process_locker::ProcessLockSharedGuard;
 
 /// Header format definition for dynamic index files (`.dixd`)
 #[repr(C)]
@@ -480,7 +479,7 @@ impl<R: ReadChunk> ReadAt for LocalDynamicReadAt<R> {
 /// Create dynamic index files (`.dixd`)
 pub struct DynamicIndexWriter {
     store: Arc<ChunkStore>,
-    _lock: tools::ProcessLockSharedGuard,
+    _lock: ProcessLockSharedGuard,
     writer: BufWriter<File>,
     closed: bool,
     filename: PathBuf,

src/backup/fixed_index.rs

@@ -1,18 +1,17 @@
-use anyhow::{bail, format_err, Error};
-use std::io::{Seek, SeekFrom};
-
-use super::chunk_stat::*;
-use super::chunk_store::*;
-use super::{ChunkReadInfo, IndexFile};
-use crate::tools;
-
 use std::fs::File;
 use std::io::Write;
 use std::os::unix::io::AsRawFd;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
+use std::io::{Seek, SeekFrom};
 
-use super::ChunkInfo;
+use anyhow::{bail, format_err, Error};
+
+use pbs_datastore::chunk_stat::ChunkStat;
+use pbs_datastore::chunk_store::ChunkStore;
+use pbs_datastore::data_blob::ChunkInfo;
+use pbs_datastore::index::{ChunkReadInfo, IndexFile};
+use pbs_tools::process_locker::ProcessLockSharedGuard;
 
 use proxmox::tools::io::ReadExt;
 use proxmox::tools::Uuid;
@@ -229,7 +228,7 @@ impl IndexFile for FixedIndexReader {
 pub struct FixedIndexWriter {
     store: Arc<ChunkStore>,
     file: File,
-    _lock: tools::ProcessLockSharedGuard,
+    _lock: ProcessLockSharedGuard,
     filename: PathBuf,
     tmp_filename: PathBuf,
     chunk_size: usize,

src/backup/mod.rs

@@ -149,12 +149,16 @@ pub const CATALOG_NAME: &str = "catalog.pcat1.didx";
 
 #[macro_export]
 macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 {
-    () =>  { "proxmox-backup-protocol-v1" }
+    () => {
+        "proxmox-backup-protocol-v1"
+    };
 }
 
 #[macro_export]
 macro_rules! PROXMOX_BACKUP_READER_PROTOCOL_ID_V1 {
-    () =>  { "proxmox-backup-reader-protocol-v1" }
+    () => {
+        "proxmox-backup-reader-protocol-v1"
+    };
 }
 
 /// Unix system user used by proxmox-backup-proxy
@@ -186,6 +190,8 @@ pub use pbs_datastore::checksum_reader;
 pub use pbs_datastore::checksum_reader::*;
 pub use pbs_datastore::checksum_writer;
 pub use pbs_datastore::checksum_writer::*;
+pub use pbs_datastore::chunk_stat;
+pub use pbs_datastore::chunk_stat::*;
 pub use pbs_datastore::chunk_store;
 pub use pbs_datastore::chunk_store::*;
 pub use pbs_datastore::chunker;
@@ -210,13 +216,12 @@ pub use pbs_datastore::key_derivation;
 pub use pbs_datastore::key_derivation::*;
 pub use pbs_datastore::manifest;
 pub use pbs_datastore::manifest::*;
+pub use pbs_datastore::read_chunk::*;
 
 mod chunk_stream;
 pub use chunk_stream::*;
 
-mod chunk_stat;
-pub use chunk_stat::*;
-
+// Split
 mod read_chunk;
 pub use read_chunk::*;
 
@@ -238,6 +243,7 @@ pub use store_progress::*;
 mod verify;
 pub use verify::*;
 
+// Move to client
 mod catalog_shell;
 pub use catalog_shell::*;
 

src/backup/read_chunk.rs

@@ -4,19 +4,12 @@ use std::sync::Arc;
 
 use anyhow::{bail, Error};
 
-use super::crypt_config::{CryptConfig, CryptMode};
-use super::data_blob::DataBlob;
+use pbs_datastore::crypt_config::{CryptConfig, CryptMode};
+use pbs_datastore::data_blob::DataBlob;
+use pbs_datastore::read_chunk::{ReadChunk, AsyncReadChunk};
+
 use super::datastore::DataStore;
 
-/// The ReadChunk trait allows reading backup data chunks (local or remote)
-pub trait ReadChunk {
-    /// Returns the encoded chunk data
-    fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error>;
-
-    /// Returns the decoded chunk data
-    fn read_chunk(&self, digest: &[u8; 32]) -> Result<Vec<u8>, Error>;
-}
-
 #[derive(Clone)]
 pub struct LocalChunkReader {
     store: Arc<DataStore>,
@@ -67,20 +60,6 @@ impl ReadChunk for LocalChunkReader {
     }
 }
 
-pub trait AsyncReadChunk: Send {
-    /// Returns the encoded chunk data
-    fn read_raw_chunk<'a>(
-        &'a self,
-        digest: &'a [u8; 32],
-    ) -> Pin<Box<dyn Future<Output = Result<DataBlob, Error>> + Send + 'a>>;
-
-    /// Returns the decoded chunk data
-    fn read_chunk<'a>(
-        &'a self,
-        digest: &'a [u8; 32],
-    ) -> Pin<Box<dyn Future<Output = Result<Vec<u8>, Error>> + Send + 'a>>;
-}
-
 impl AsyncReadChunk for LocalChunkReader {
     fn read_raw_chunk<'a>(
         &'a self,