move chunk_stat, read_chunk to pbs-datastore

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>

pbs-datastore/src/lib.rs

@@ -182,6 +182,7 @@ pub mod backup_info;
 pub mod catalog;
 pub mod checksum_reader;
 pub mod checksum_writer;
+pub mod chunk_stat;
 pub mod chunk_store;
 pub mod chunker;
 pub mod crypt_config;
@@ -194,6 +195,7 @@ pub mod file_formats;
 pub mod index;
 pub mod key_derivation;
 pub mod manifest;
+pub mod read_chunk;
 pub mod task;
 
 pub use backup_info::{BackupDir, BackupGroup, BackupInfo};

pbs-datastore/src/read_chunk.rs

@@ -0,0 +1,29 @@
+use std::future::Future;
+use std::pin::Pin;
+
+use anyhow::Error;
+
+use crate::data_blob::DataBlob;
+
+/// The ReadChunk trait allows reading backup data chunks (local or remote)
+pub trait ReadChunk {
+    /// Returns the encoded chunk data
+    fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error>;
+
+    /// Returns the decoded chunk data
+    fn read_chunk(&self, digest: &[u8; 32]) -> Result<Vec<u8>, Error>;
+}
+
+pub trait AsyncReadChunk: Send {
+    /// Returns the encoded chunk data
+    fn read_raw_chunk<'a>(
+        &'a self,
+        digest: &'a [u8; 32],
+    ) -> Pin<Box<dyn Future<Output = Result<DataBlob, Error>> + Send + 'a>>;
+
+    /// Returns the decoded chunk data
+    fn read_chunk<'a>(
+        &'a self,
+        digest: &'a [u8; 32],
+    ) -> Pin<Box<dyn Future<Output = Result<Vec<u8>, Error>> + Send + 'a>>;
+}

src/backup/cached_chunk_reader.rs

@@ -1,20 +1,22 @@
 //! An async and concurrency safe data reader backed by a local LRU cache.
 
-use anyhow::Error;
+use std::future::Future;
-use futures::future::Future;
-use futures::ready;
-use tokio::io::{AsyncRead, AsyncSeek, ReadBuf};
-
 use std::io::SeekFrom;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
 
-use super::{AsyncReadChunk, IndexFile};
+use anyhow::Error;
-use crate::tools::async_lru_cache::{AsyncCacher, AsyncLruCache};
+use futures::ready;
+use tokio::io::{AsyncRead, AsyncSeek, ReadBuf};
+
 use proxmox::io_format_err;
 use proxmox::sys::error::io_err_other;
 
+use pbs_datastore::read_chunk::AsyncReadChunk;
+use super::IndexFile;
+use crate::tools::async_lru_cache::{AsyncCacher, AsyncLruCache};
+
 struct AsyncChunkCacher<T> {
     reader: Arc<T>,
 }

src/backup/dynamic_index.rs

@@ -14,14 +14,13 @@ use proxmox::tools::uuid::Uuid;
 use proxmox::tools::mmap::Mmap;
 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
 
-use super::chunk_stat::ChunkStat;
+use pbs_datastore::Chunker;
-use super::chunk_store::ChunkStore;
+use pbs_datastore::index::{IndexFile, ChunkReadInfo};
-use super::index::ChunkReadInfo;
+use pbs_datastore::chunk_stat::ChunkStat;
-use super::read_chunk::ReadChunk;
+use pbs_datastore::data_blob::{DataBlob, DataChunkBuilder};
-use super::Chunker;
+use pbs_datastore::chunk_store::ChunkStore;
-use super::IndexFile;
+use pbs_datastore::read_chunk::ReadChunk;
-use super::{DataBlob, DataChunkBuilder};
+use pbs_tools::process_locker::ProcessLockSharedGuard;
-use crate::tools;
 
 /// Header format definition for dynamic index files (`.dixd`)
 #[repr(C)]
@@ -480,7 +479,7 @@ impl<R: ReadChunk> ReadAt for LocalDynamicReadAt<R> {
 /// Create dynamic index files (`.dixd`)
 pub struct DynamicIndexWriter {
     store: Arc<ChunkStore>,
-    _lock: tools::ProcessLockSharedGuard,
+    _lock: ProcessLockSharedGuard,
     writer: BufWriter<File>,
     closed: bool,
     filename: PathBuf,

src/backup/fixed_index.rs

@@ -1,18 +1,17 @@
-use anyhow::{bail, format_err, Error};
-use std::io::{Seek, SeekFrom};
-
-use super::chunk_stat::*;
-use super::chunk_store::*;
-use super::{ChunkReadInfo, IndexFile};
-use crate::tools;
-
 use std::fs::File;
 use std::io::Write;
 use std::os::unix::io::AsRawFd;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
+use std::io::{Seek, SeekFrom};
 
-use super::ChunkInfo;
+use anyhow::{bail, format_err, Error};
+
+use pbs_datastore::chunk_stat::ChunkStat;
+use pbs_datastore::chunk_store::ChunkStore;
+use pbs_datastore::data_blob::ChunkInfo;
+use pbs_datastore::index::{ChunkReadInfo, IndexFile};
+use pbs_tools::process_locker::ProcessLockSharedGuard;
 
 use proxmox::tools::io::ReadExt;
 use proxmox::tools::Uuid;
@@ -229,7 +228,7 @@ impl IndexFile for FixedIndexReader {
 pub struct FixedIndexWriter {
     store: Arc<ChunkStore>,
     file: File,
-    _lock: tools::ProcessLockSharedGuard,
+    _lock: ProcessLockSharedGuard,
     filename: PathBuf,
     tmp_filename: PathBuf,
     chunk_size: usize,

src/backup/mod.rs

@@ -149,12 +149,16 @@ pub const CATALOG_NAME: &str = "catalog.pcat1.didx";
 
 #[macro_export]
 macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 {
-    () =>  { "proxmox-backup-protocol-v1" }
+    () => {
+        "proxmox-backup-protocol-v1"
+    };
 }
 
 #[macro_export]
 macro_rules! PROXMOX_BACKUP_READER_PROTOCOL_ID_V1 {
-    () =>  { "proxmox-backup-reader-protocol-v1" }
+    () => {
+        "proxmox-backup-reader-protocol-v1"
+    };
 }
 
 /// Unix system user used by proxmox-backup-proxy
@@ -186,6 +190,8 @@ pub use pbs_datastore::checksum_reader;
 pub use pbs_datastore::checksum_reader::*;
 pub use pbs_datastore::checksum_writer;
 pub use pbs_datastore::checksum_writer::*;
+pub use pbs_datastore::chunk_stat;
+pub use pbs_datastore::chunk_stat::*;
 pub use pbs_datastore::chunk_store;
 pub use pbs_datastore::chunk_store::*;
 pub use pbs_datastore::chunker;
@@ -210,13 +216,12 @@ pub use pbs_datastore::key_derivation;
 pub use pbs_datastore::key_derivation::*;
 pub use pbs_datastore::manifest;
 pub use pbs_datastore::manifest::*;
+pub use pbs_datastore::read_chunk::*;
 
 mod chunk_stream;
 pub use chunk_stream::*;
 
-mod chunk_stat;
+// Split
-pub use chunk_stat::*;
-
 mod read_chunk;
 pub use read_chunk::*;
 
@@ -238,6 +243,7 @@ pub use store_progress::*;
 mod verify;
 pub use verify::*;
 
+// Move to client
 mod catalog_shell;
 pub use catalog_shell::*;
 

src/backup/read_chunk.rs

@@ -4,19 +4,12 @@ use std::sync::Arc;
 
 use anyhow::{bail, Error};
 
-use super::crypt_config::{CryptConfig, CryptMode};
+use pbs_datastore::crypt_config::{CryptConfig, CryptMode};
-use super::data_blob::DataBlob;
+use pbs_datastore::data_blob::DataBlob;
+use pbs_datastore::read_chunk::{ReadChunk, AsyncReadChunk};
+
 use super::datastore::DataStore;
 
-/// The ReadChunk trait allows reading backup data chunks (local or remote)
-pub trait ReadChunk {
-    /// Returns the encoded chunk data
-    fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error>;
-
-    /// Returns the decoded chunk data
-    fn read_chunk(&self, digest: &[u8; 32]) -> Result<Vec<u8>, Error>;
-}
-
 #[derive(Clone)]
 pub struct LocalChunkReader {
     store: Arc<DataStore>,
@@ -67,20 +60,6 @@ impl ReadChunk for LocalChunkReader {
     }
 }
 
-pub trait AsyncReadChunk: Send {
-    /// Returns the encoded chunk data
-    fn read_raw_chunk<'a>(
-        &'a self,
-        digest: &'a [u8; 32],
-    ) -> Pin<Box<dyn Future<Output = Result<DataBlob, Error>> + Send + 'a>>;
-
-    /// Returns the decoded chunk data
-    fn read_chunk<'a>(
-        &'a self,
-        digest: &'a [u8; 32],
-    ) -> Pin<Box<dyn Future<Output = Result<Vec<u8>, Error>> + Send + 'a>>;
-}
-
 impl AsyncReadChunk for LocalChunkReader {
     fn read_raw_chunk<'a>(
         &'a self,