move src/backup/datastore.rs into pbs_datastore crate

pbs-datastore/Cargo.toml

@@ -11,6 +11,7 @@ base64 = "0.12"
 crc32fast = "1"
 endian_trait = { version = "0.6", features = [ "arrays" ] }
 futures = "0.3"
+lazy_static = "1.4"
 libc = "0.2"
 log = "0.4"
 nix = "0.19.1"
@@ -18,6 +19,7 @@ openssl = "0.10"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 tokio = { version = "1.6", features = [] }
+walkdir = "2"
 zstd = { version = "0.6", features = [ "bindgen" ] }
 
 pathpatterns = "0.1.2"

pbs-datastore/src/datastore.rs

@@ -12,23 +12,24 @@ use lazy_static::lazy_static;
 use proxmox::tools::fs::{replace_file, file_read_optional_string, CreateOptions};
 
 use pbs_api_types::{UPID, DataStoreConfig, Authid, GarbageCollectionStatus};
-use pbs_datastore::DataBlob;
-use pbs_datastore::backup_info::{BackupGroup, BackupDir};
-use pbs_datastore::chunk_store::ChunkStore;
-use pbs_datastore::dynamic_index::{DynamicIndexReader, DynamicIndexWriter};
-use pbs_datastore::fixed_index::{FixedIndexReader, FixedIndexWriter};
-use pbs_datastore::index::IndexFile;
-use pbs_datastore::manifest::{
-    MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME, CLIENT_LOG_BLOB_NAME,
-    ArchiveType, BackupManifest,
-    archive_type,
-};
 use pbs_tools::format::HumanByte;
 use pbs_tools::fs::{lock_dir_noblock, DirLockGuard};
 use pbs_tools::process_locker::ProcessLockSharedGuard;
 use pbs_tools::{task_log, task_warn, task::WorkerTaskContext};
 use pbs_config::{open_backup_lockfile, BackupLockGuard};
 
+use crate::DataBlob;
+use crate::backup_info::{BackupGroup, BackupDir};
+use crate::chunk_store::ChunkStore;
+use crate::dynamic_index::{DynamicIndexReader, DynamicIndexWriter};
+use crate::fixed_index::{FixedIndexReader, FixedIndexWriter};
+use crate::index::IndexFile;
+use crate::manifest::{
+    MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME, CLIENT_LOG_BLOB_NAME,
+    ArchiveType, BackupManifest,
+    archive_type,
+};
+
 lazy_static! {
     static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new());
 }

pbs-datastore/src/lib.rs

@@ -195,3 +195,6 @@ pub use data_blob_reader::DataBlobReader;
 pub use data_blob_writer::DataBlobWriter;
 pub use manifest::BackupManifest;
 pub use store_progress::StoreProgress;
+
+mod datastore;
+pub use datastore::{check_backup_owner, DataStore};

src/api2/admin/datastore.rs

@@ -39,7 +39,7 @@ use pbs_api_types::{ Authid, BackupContent, Counts, CryptMode,
 
 };
 use pbs_client::pxar::create_zip;
-use pbs_datastore::{BackupDir, BackupGroup, StoreProgress, CATALOG_NAME};
+use pbs_datastore::{check_backup_owner, DataStore, BackupDir, BackupGroup, StoreProgress, CATALOG_NAME};
 use pbs_datastore::backup_info::BackupInfo;
 use pbs_datastore::cached_chunk_reader::CachedChunkReader;
 use pbs_datastore::catalog::{ArchiveEntry, CatalogReader};
@@ -59,8 +59,8 @@ use proxmox_rest_server::{WorkerTask, formatter};
 
 use crate::api2::node::rrd::create_value_from_rrd;
 use crate::backup::{
-    check_backup_owner, verify_all_backups, verify_backup_group, verify_backup_dir, verify_filter,
+    verify_all_backups, verify_backup_group, verify_backup_dir, verify_filter,
-    DataStore, LocalChunkReader,
+    LocalChunkReader,
 };
 
 use crate::server::jobstate::Job;

src/api2/backup/environment.rs

@@ -10,14 +10,14 @@ use proxmox::tools::digest_to_hex;
 use proxmox::tools::fs::{replace_file, CreateOptions};
 use proxmox::api::{RpcEnvironment, RpcEnvironmentType};
 
-use pbs_datastore::DataBlob;
+use pbs_datastore::{DataStore, DataBlob};
 use pbs_datastore::backup_info::{BackupDir, BackupInfo};
 use pbs_datastore::dynamic_index::DynamicIndexWriter;
 use pbs_datastore::fixed_index::FixedIndexWriter;
 use pbs_api_types::Authid;
 use proxmox_rest_server::{WorkerTask, formatter::*};
 
-use crate::backup::{verify_backup_dir_with_lock, DataStore};
+use crate::backup::verify_backup_dir_with_lock;
 
 use hyper::{Body, Response};
 

src/api2/backup/mod.rs

@@ -19,15 +19,13 @@ use pbs_api_types::{
 };
 use pbs_tools::fs::lock_dir_noblock_shared;
 use pbs_tools::json::{required_array_param, required_integer_param, required_string_param};
-use pbs_datastore::PROXMOX_BACKUP_PROTOCOL_ID_V1;
+use pbs_config::CachedUserInfo;
+use pbs_datastore::{DataStore, PROXMOX_BACKUP_PROTOCOL_ID_V1};
 use pbs_datastore::backup_info::{BackupDir, BackupGroup, BackupInfo};
 use pbs_datastore::index::IndexFile;
 use pbs_datastore::manifest::{archive_type, ArchiveType};
 use proxmox_rest_server::{WorkerTask, H2Service};
 
-use crate::backup::DataStore;
-use pbs_config::CachedUserInfo;
-
 mod environment;
 use environment::*;
 

src/api2/backup/upload_chunk.rs

@@ -12,13 +12,11 @@ use proxmox::{sortable, identity};
 use proxmox::api::{ApiResponseFuture, ApiHandler, ApiMethod, RpcEnvironment};
 use proxmox::api::schema::*;
 
-use pbs_datastore::DataBlob;
+use pbs_datastore::{DataStore, DataBlob};
 use pbs_datastore::file_formats::{DataBlobHeader, EncryptedDataBlobHeader};
 use pbs_tools::json::{required_integer_param, required_string_param};
 use pbs_api_types::{CHUNK_DIGEST_SCHEMA, BACKUP_ARCHIVE_NAME_SCHEMA};
 
-use crate::backup::DataStore;
-
 use super::environment::*;
 
 pub struct UploadChunk {

src/api2/pull.rs

@@ -16,9 +16,9 @@ use pbs_api_types::{
 use pbs_tools::task_log;
 use proxmox_rest_server::WorkerTask;
 use pbs_config::CachedUserInfo;
+use pbs_datastore::DataStore;
 
 use crate::server::{jobstate::Job, pull::pull_store};
-use crate::backup::DataStore;
 
 pub fn check_pull_privs(
     auth_id: &Authid,

src/api2/reader/environment.rs

@@ -6,14 +6,11 @@ use serde_json::{json, Value};
 use proxmox::api::{RpcEnvironment, RpcEnvironmentType};
 
 use pbs_datastore::backup_info::BackupDir;
+use pbs_datastore::DataStore;
 use pbs_api_types::Authid;
 use proxmox_rest_server::formatter::*;
-
-use crate::backup::DataStore;
 use proxmox_rest_server::WorkerTask;
 
-//use proxmox::tools;
-
 /// `RpcEnvironmet` implementation for backup reader service
 #[derive(Clone)]
 pub struct ReaderEnvironment {

src/api2/reader/mod.rs

@@ -34,14 +34,14 @@ use pbs_api_types::{
 };
 use pbs_tools::fs::lock_dir_noblock_shared;
 use pbs_tools::json::{required_integer_param, required_string_param};
-use pbs_datastore::PROXMOX_BACKUP_READER_PROTOCOL_ID_V1;
+use pbs_datastore::{DataStore, PROXMOX_BACKUP_READER_PROTOCOL_ID_V1};
 use pbs_datastore::backup_info::BackupDir;
 use pbs_datastore::index::IndexFile;
 use pbs_datastore::manifest::{archive_type, ArchiveType};
 use pbs_config::CachedUserInfo;
 use proxmox_rest_server::{WorkerTask, H2Service};
 
-use crate::{api2::helpers, backup::DataStore};
+use crate::api2::helpers;
 
 mod environment;
 use environment::*;

src/api2/status.rs

@@ -18,11 +18,11 @@ use pbs_api_types::{
     DATASTORE_SCHEMA, RRDMode, RRDTimeFrameResolution, Authid,
     PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_BACKUP,
 };
-
+use pbs_datastore::DataStore;
-use crate::backup::DataStore;
-use crate::tools::statistics::{linear_regression};
 use pbs_config::CachedUserInfo;
 
+use crate::tools::statistics::{linear_regression};
+
 #[api(
     returns: {
         description: "Lists the Status of the Datastores.",

src/api2/tape/backup.rs

@@ -20,7 +20,7 @@ use pbs_api_types::{
     UPID_SCHEMA, JOB_ID_SCHEMA, PRIV_DATASTORE_READ, PRIV_TAPE_AUDIT, PRIV_TAPE_WRITE,
 };
 
-use pbs_datastore::StoreProgress;
+use pbs_datastore::{DataStore, StoreProgress};
 use pbs_datastore::backup_info::{BackupDir, BackupInfo};
 use pbs_tools::{task_log, task_warn, task::WorkerTaskContext};
 use pbs_config::CachedUserInfo;
@@ -36,7 +36,7 @@ use crate::{
             compute_schedule_status,
         },
     },
-    backup::{DataStore, SnapshotReader},
+    backup::SnapshotReader,
     tape::{
         TAPE_STATUS_DIR,
         Inventory,

src/api2/tape/restore.rs

@@ -34,7 +34,7 @@ use pbs_api_types::{
     UPID_SCHEMA, TAPE_RESTORE_SNAPSHOT_SCHEMA,
     PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_MODIFY, PRIV_TAPE_READ,
 };
-use pbs_datastore::DataBlob;
+use pbs_datastore::{DataStore, DataBlob};
 use pbs_datastore::backup_info::BackupDir;
 use pbs_datastore::dynamic_index::DynamicIndexReader;
 use pbs_datastore::fixed_index::FixedIndexReader;
@@ -50,7 +50,6 @@ use proxmox_rest_server::WorkerTask;
 
 use crate::{
     tools::ParallelHandler,
-    backup::DataStore,
     server::lookup_user_email,
     tape::{
         TAPE_STATUS_DIR,

src/backup/mod.rs

@@ -7,9 +7,6 @@ pub const CATALOG_NAME: &str = "catalog.pcat1.didx";
 mod read_chunk;
 pub use read_chunk::*;
 
-mod datastore;
-pub use datastore::*;
-
 mod verify;
 pub use verify::*;
 

src/backup/read_chunk.rs

@@ -8,8 +8,7 @@ use pbs_tools::crypt_config::CryptConfig;
 use pbs_api_types::CryptMode;
 use pbs_datastore::data_blob::DataBlob;
 use pbs_datastore::read_chunk::{ReadChunk, AsyncReadChunk};
-
+use pbs_datastore::DataStore;
-use super::datastore::DataStore;
 
 #[derive(Clone)]
 pub struct LocalChunkReader {

src/backup/snapshot_reader.rs

@@ -11,10 +11,9 @@ use pbs_datastore::index::IndexFile;
 use pbs_datastore::fixed_index::FixedIndexReader;
 use pbs_datastore::dynamic_index::DynamicIndexReader;
 use pbs_datastore::manifest::{archive_type, ArchiveType, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME};
+use pbs_datastore::DataStore;
 use pbs_tools::fs::lock_dir_noblock_shared;
 
-use crate::backup::DataStore;
-
 /// Helper to access the contents of a datastore backup snapshot
 ///
 /// This make it easy to iterate over all used chunks and files.

src/backup/verify.rs

@@ -7,17 +7,14 @@ use std::time::Instant;
 use anyhow::{bail, format_err, Error};
 
 use pbs_api_types::{Authid, CryptMode, VerifyState, UPID, SnapshotVerifyState};
-use pbs_datastore::{DataBlob, StoreProgress};
+use pbs_datastore::{DataStore, DataBlob, StoreProgress};
 use pbs_datastore::backup_info::{BackupGroup, BackupDir, BackupInfo};
 use pbs_datastore::index::IndexFile;
 use pbs_datastore::manifest::{archive_type, ArchiveType, BackupManifest, FileInfo};
 use pbs_tools::fs::lock_dir_noblock_shared;
 use pbs_tools::{task_log, task::WorkerTaskContext};
 
-use crate::{
+use crate::tools::ParallelHandler;
-    backup::DataStore,
-    tools::ParallelHandler,
-};
 
 /// A VerifyWorker encapsulates a task worker, datastore and information about which chunks have
 /// already been verified or detected as corrupt.

src/bin/proxmox-backup-proxy.rs

@@ -19,11 +19,11 @@ use proxmox::api::RpcEnvironmentType;
 use proxmox::sys::linux::socket::set_tcp_keepalive;
 use proxmox::tools::fs::CreateOptions;
 
-use proxmox_rest_server::{rotate_task_log_archive, ApiConfig, RestServer, WorkerTask};
 use pbs_tools::task_log;
+use pbs_datastore::DataStore;
+use proxmox_rest_server::{rotate_task_log_archive, ApiConfig, RestServer, WorkerTask};
 
 use proxmox_backup::{
-    backup::DataStore,
     server::{
         auth::default_api_auth,
         jobstate::{
@@ -238,7 +238,7 @@ async fn run() -> Result<(), Error> {
     commando_sock.register_command(
         "datastore-removed".to_string(),
         |_value| {
-            if let Err(err) = proxmox_backup::backup::DataStore::remove_unused_datastores() {
+            if let Err(err) = DataStore::remove_unused_datastores() {
                 log::error!("could not refresh datastores: {}", err);
             }
             Ok(Value::Null)

src/server/gc_job.rs

@@ -3,12 +3,10 @@ use anyhow::Error;
 
 use pbs_api_types::Authid;
 use pbs_tools::task_log;
+use pbs_datastore::DataStore;
 use proxmox_rest_server::WorkerTask;
 
-use crate::{
+use crate::server::jobstate::Job;
-    server::jobstate::Job,
-    backup::DataStore,
-};
 
 /// Runs a garbage collection job.
 pub fn do_garbage_collection_job(

src/server/prune_job.rs

@@ -4,15 +4,13 @@ use anyhow::Error;
 
 use pbs_datastore::backup_info::BackupInfo;
 use pbs_datastore::prune::compute_prune_info;
+use pbs_datastore::DataStore;
 use pbs_api_types::{Authid, PRIV_DATASTORE_MODIFY, PruneOptions};
 use pbs_config::CachedUserInfo;
 use pbs_tools::{task_log, task_warn};
 use proxmox_rest_server::WorkerTask;
 
-use crate::{
+use crate::server::jobstate::Job;
-    backup::DataStore,
-    server::jobstate::Job,
- };
 
 pub fn prune_datastore(
     worker: Arc<WorkerTask>,

src/server/pull.rs

@@ -13,7 +13,7 @@ use serde_json::json;
 use proxmox::api::error::{HttpError, StatusCode};
 
 use pbs_api_types::{Authid, SnapshotListItem, GroupListItem};
-use pbs_datastore::{BackupInfo, BackupDir, BackupGroup, StoreProgress};
+use pbs_datastore::{DataStore, BackupInfo, BackupDir, BackupGroup, StoreProgress};
 use pbs_datastore::data_blob::DataBlob;
 use pbs_datastore::dynamic_index::DynamicIndexReader;
 use pbs_datastore::fixed_index::FixedIndexReader;
@@ -26,10 +26,7 @@ use pbs_tools::task_log;
 use pbs_client::{BackupReader, BackupRepository, HttpClient, HttpClientOptions, RemoteChunkReader};
 use proxmox_rest_server::WorkerTask;
 
-use crate::{
+use crate::tools::ParallelHandler;
-    backup::DataStore,
-    tools::ParallelHandler,
-};
 
 // fixme: implement filters
 // fixme: delete vanished groups

src/server/verify_job.rs

@@ -3,11 +3,11 @@ use anyhow::{format_err, Error};
 use pbs_tools::task_log;
 use pbs_api_types::{Authid, VerificationJobConfig};
 use proxmox_rest_server::WorkerTask;
+use pbs_datastore::DataStore;
 
 use crate::{
     server::jobstate::Job,
     backup::{
-        DataStore,
         verify_filter,
         verify_all_backups,
     },

src/tape/pool_writer/mod.rs

@@ -19,10 +19,11 @@ use pbs_tape::{
     TapeWrite,
     sg_tape::tape_alert_flags_critical,
 };
+use pbs_datastore::DataStore;
 use proxmox_rest_server::WorkerTask;
 
 use crate::{
-    backup::{DataStore, SnapshotReader},
+    backup::SnapshotReader,
     tape::{
         TAPE_STATUS_DIR,
         MAX_CHUNK_ARCHIVE_SIZE,

src/tape/pool_writer/new_chunks_iterator.rs

@@ -3,9 +3,9 @@ use std::sync::{Arc, Mutex};
 
 use anyhow::{format_err, Error};
 
-use pbs_datastore::DataBlob;
+use pbs_datastore::{DataStore, DataBlob};
 
-use crate::backup::{DataStore, SnapshotReader};
+use crate::backup::SnapshotReader;
 use crate::tape::CatalogSet;
 
 /// Chunk iterator which use a separate thread to read chunks