diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml index 578b8689..077ead47 100644 --- a/pbs-datastore/Cargo.toml +++ b/pbs-datastore/Cargo.toml @@ -11,6 +11,7 @@ base64 = "0.12" crc32fast = "1" endian_trait = { version = "0.6", features = [ "arrays" ] } futures = "0.3" +lazy_static = "1.4" libc = "0.2" log = "0.4" nix = "0.19.1" @@ -18,6 +19,7 @@ openssl = "0.10" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1.6", features = [] } +walkdir = "2" zstd = { version = "0.6", features = [ "bindgen" ] } pathpatterns = "0.1.2" diff --git a/src/backup/datastore.rs b/pbs-datastore/src/datastore.rs similarity index 98% rename from src/backup/datastore.rs rename to pbs-datastore/src/datastore.rs index 881c67ff..b068a9c9 100644 --- a/src/backup/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -12,23 +12,24 @@ use lazy_static::lazy_static; use proxmox::tools::fs::{replace_file, file_read_optional_string, CreateOptions}; use pbs_api_types::{UPID, DataStoreConfig, Authid, GarbageCollectionStatus}; -use pbs_datastore::DataBlob; -use pbs_datastore::backup_info::{BackupGroup, BackupDir}; -use pbs_datastore::chunk_store::ChunkStore; -use pbs_datastore::dynamic_index::{DynamicIndexReader, DynamicIndexWriter}; -use pbs_datastore::fixed_index::{FixedIndexReader, FixedIndexWriter}; -use pbs_datastore::index::IndexFile; -use pbs_datastore::manifest::{ - MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME, CLIENT_LOG_BLOB_NAME, - ArchiveType, BackupManifest, - archive_type, -}; use pbs_tools::format::HumanByte; use pbs_tools::fs::{lock_dir_noblock, DirLockGuard}; use pbs_tools::process_locker::ProcessLockSharedGuard; use pbs_tools::{task_log, task_warn, task::WorkerTaskContext}; use pbs_config::{open_backup_lockfile, BackupLockGuard}; +use crate::DataBlob; +use crate::backup_info::{BackupGroup, BackupDir}; +use crate::chunk_store::ChunkStore; +use crate::dynamic_index::{DynamicIndexReader, DynamicIndexWriter}; +use crate::fixed_index::{FixedIndexReader, FixedIndexWriter}; +use crate::index::IndexFile; +use crate::manifest::{ + MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME, CLIENT_LOG_BLOB_NAME, + ArchiveType, BackupManifest, + archive_type, +}; + lazy_static! { static ref DATASTORE_MAP: Mutex>> = Mutex::new(HashMap::new()); } diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs index 5a09666b..fc08d00c 100644 --- a/pbs-datastore/src/lib.rs +++ b/pbs-datastore/src/lib.rs @@ -195,3 +195,6 @@ pub use data_blob_reader::DataBlobReader; pub use data_blob_writer::DataBlobWriter; pub use manifest::BackupManifest; pub use store_progress::StoreProgress; + +mod datastore; +pub use datastore::{check_backup_owner, DataStore}; diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index a4152905..77e24513 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -39,7 +39,7 @@ use pbs_api_types::{ Authid, BackupContent, Counts, CryptMode, }; use pbs_client::pxar::create_zip; -use pbs_datastore::{BackupDir, BackupGroup, StoreProgress, CATALOG_NAME}; +use pbs_datastore::{check_backup_owner, DataStore, BackupDir, BackupGroup, StoreProgress, CATALOG_NAME}; use pbs_datastore::backup_info::BackupInfo; use pbs_datastore::cached_chunk_reader::CachedChunkReader; use pbs_datastore::catalog::{ArchiveEntry, CatalogReader}; @@ -59,8 +59,8 @@ use proxmox_rest_server::{WorkerTask, formatter}; use crate::api2::node::rrd::create_value_from_rrd; use crate::backup::{ - check_backup_owner, verify_all_backups, verify_backup_group, verify_backup_dir, verify_filter, - DataStore, LocalChunkReader, + verify_all_backups, verify_backup_group, verify_backup_dir, verify_filter, + LocalChunkReader, }; use crate::server::jobstate::Job; diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs index 7842df44..a1c4186b 100644 --- a/src/api2/backup/environment.rs +++ b/src/api2/backup/environment.rs @@ -10,14 +10,14 @@ use proxmox::tools::digest_to_hex; use proxmox::tools::fs::{replace_file, CreateOptions}; use proxmox::api::{RpcEnvironment, RpcEnvironmentType}; -use pbs_datastore::DataBlob; +use pbs_datastore::{DataStore, DataBlob}; use pbs_datastore::backup_info::{BackupDir, BackupInfo}; use pbs_datastore::dynamic_index::DynamicIndexWriter; use pbs_datastore::fixed_index::FixedIndexWriter; use pbs_api_types::Authid; use proxmox_rest_server::{WorkerTask, formatter::*}; -use crate::backup::{verify_backup_dir_with_lock, DataStore}; +use crate::backup::verify_backup_dir_with_lock; use hyper::{Body, Response}; diff --git a/src/api2/backup/mod.rs b/src/api2/backup/mod.rs index d9ad95de..4432c8d5 100644 --- a/src/api2/backup/mod.rs +++ b/src/api2/backup/mod.rs @@ -19,15 +19,13 @@ use pbs_api_types::{ }; use pbs_tools::fs::lock_dir_noblock_shared; use pbs_tools::json::{required_array_param, required_integer_param, required_string_param}; -use pbs_datastore::PROXMOX_BACKUP_PROTOCOL_ID_V1; +use pbs_config::CachedUserInfo; +use pbs_datastore::{DataStore, PROXMOX_BACKUP_PROTOCOL_ID_V1}; use pbs_datastore::backup_info::{BackupDir, BackupGroup, BackupInfo}; use pbs_datastore::index::IndexFile; use pbs_datastore::manifest::{archive_type, ArchiveType}; use proxmox_rest_server::{WorkerTask, H2Service}; -use crate::backup::DataStore; -use pbs_config::CachedUserInfo; - mod environment; use environment::*; diff --git a/src/api2/backup/upload_chunk.rs b/src/api2/backup/upload_chunk.rs index 4a20b867..abf20888 100644 --- a/src/api2/backup/upload_chunk.rs +++ b/src/api2/backup/upload_chunk.rs @@ -12,13 +12,11 @@ use proxmox::{sortable, identity}; use proxmox::api::{ApiResponseFuture, ApiHandler, ApiMethod, RpcEnvironment}; use proxmox::api::schema::*; -use pbs_datastore::DataBlob; +use pbs_datastore::{DataStore, DataBlob}; use pbs_datastore::file_formats::{DataBlobHeader, EncryptedDataBlobHeader}; use pbs_tools::json::{required_integer_param, required_string_param}; use pbs_api_types::{CHUNK_DIGEST_SCHEMA, BACKUP_ARCHIVE_NAME_SCHEMA}; -use crate::backup::DataStore; - use super::environment::*; pub struct UploadChunk { diff --git a/src/api2/pull.rs b/src/api2/pull.rs index 8a99a9f6..ea8faab8 100644 --- a/src/api2/pull.rs +++ b/src/api2/pull.rs @@ -16,9 +16,9 @@ use pbs_api_types::{ use pbs_tools::task_log; use proxmox_rest_server::WorkerTask; use pbs_config::CachedUserInfo; +use pbs_datastore::DataStore; use crate::server::{jobstate::Job, pull::pull_store}; -use crate::backup::DataStore; pub fn check_pull_privs( auth_id: &Authid, diff --git a/src/api2/reader/environment.rs b/src/api2/reader/environment.rs index 329a5771..394de7b2 100644 --- a/src/api2/reader/environment.rs +++ b/src/api2/reader/environment.rs @@ -6,14 +6,11 @@ use serde_json::{json, Value}; use proxmox::api::{RpcEnvironment, RpcEnvironmentType}; use pbs_datastore::backup_info::BackupDir; +use pbs_datastore::DataStore; use pbs_api_types::Authid; use proxmox_rest_server::formatter::*; - -use crate::backup::DataStore; use proxmox_rest_server::WorkerTask; -//use proxmox::tools; - /// `RpcEnvironmet` implementation for backup reader service #[derive(Clone)] pub struct ReaderEnvironment { diff --git a/src/api2/reader/mod.rs b/src/api2/reader/mod.rs index e451a596..1fd25a5c 100644 --- a/src/api2/reader/mod.rs +++ b/src/api2/reader/mod.rs @@ -34,14 +34,14 @@ use pbs_api_types::{ }; use pbs_tools::fs::lock_dir_noblock_shared; use pbs_tools::json::{required_integer_param, required_string_param}; -use pbs_datastore::PROXMOX_BACKUP_READER_PROTOCOL_ID_V1; +use pbs_datastore::{DataStore, PROXMOX_BACKUP_READER_PROTOCOL_ID_V1}; use pbs_datastore::backup_info::BackupDir; use pbs_datastore::index::IndexFile; use pbs_datastore::manifest::{archive_type, ArchiveType}; use pbs_config::CachedUserInfo; use proxmox_rest_server::{WorkerTask, H2Service}; -use crate::{api2::helpers, backup::DataStore}; +use crate::api2::helpers; mod environment; use environment::*; diff --git a/src/api2/status.rs b/src/api2/status.rs index 051da659..13d1f74e 100644 --- a/src/api2/status.rs +++ b/src/api2/status.rs @@ -18,11 +18,11 @@ use pbs_api_types::{ DATASTORE_SCHEMA, RRDMode, RRDTimeFrameResolution, Authid, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_BACKUP, }; - -use crate::backup::DataStore; -use crate::tools::statistics::{linear_regression}; +use pbs_datastore::DataStore; use pbs_config::CachedUserInfo; +use crate::tools::statistics::{linear_regression}; + #[api( returns: { description: "Lists the Status of the Datastores.", diff --git a/src/api2/tape/backup.rs b/src/api2/tape/backup.rs index c5588ddc..aad5968a 100644 --- a/src/api2/tape/backup.rs +++ b/src/api2/tape/backup.rs @@ -20,7 +20,7 @@ use pbs_api_types::{ UPID_SCHEMA, JOB_ID_SCHEMA, PRIV_DATASTORE_READ, PRIV_TAPE_AUDIT, PRIV_TAPE_WRITE, }; -use pbs_datastore::StoreProgress; +use pbs_datastore::{DataStore, StoreProgress}; use pbs_datastore::backup_info::{BackupDir, BackupInfo}; use pbs_tools::{task_log, task_warn, task::WorkerTaskContext}; use pbs_config::CachedUserInfo; @@ -36,7 +36,7 @@ use crate::{ compute_schedule_status, }, }, - backup::{DataStore, SnapshotReader}, + backup::SnapshotReader, tape::{ TAPE_STATUS_DIR, Inventory, diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs index a62bb279..61e17d1b 100644 --- a/src/api2/tape/restore.rs +++ b/src/api2/tape/restore.rs @@ -34,7 +34,7 @@ use pbs_api_types::{ UPID_SCHEMA, TAPE_RESTORE_SNAPSHOT_SCHEMA, PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_MODIFY, PRIV_TAPE_READ, }; -use pbs_datastore::DataBlob; +use pbs_datastore::{DataStore, DataBlob}; use pbs_datastore::backup_info::BackupDir; use pbs_datastore::dynamic_index::DynamicIndexReader; use pbs_datastore::fixed_index::FixedIndexReader; @@ -50,7 +50,6 @@ use proxmox_rest_server::WorkerTask; use crate::{ tools::ParallelHandler, - backup::DataStore, server::lookup_user_email, tape::{ TAPE_STATUS_DIR, diff --git a/src/backup/mod.rs b/src/backup/mod.rs index d9aacd16..861010f7 100644 --- a/src/backup/mod.rs +++ b/src/backup/mod.rs @@ -7,9 +7,6 @@ pub const CATALOG_NAME: &str = "catalog.pcat1.didx"; mod read_chunk; pub use read_chunk::*; -mod datastore; -pub use datastore::*; - mod verify; pub use verify::*; diff --git a/src/backup/read_chunk.rs b/src/backup/read_chunk.rs index 1e67b561..028db311 100644 --- a/src/backup/read_chunk.rs +++ b/src/backup/read_chunk.rs @@ -8,8 +8,7 @@ use pbs_tools::crypt_config::CryptConfig; use pbs_api_types::CryptMode; use pbs_datastore::data_blob::DataBlob; use pbs_datastore::read_chunk::{ReadChunk, AsyncReadChunk}; - -use super::datastore::DataStore; +use pbs_datastore::DataStore; #[derive(Clone)] pub struct LocalChunkReader { diff --git a/src/backup/snapshot_reader.rs b/src/backup/snapshot_reader.rs index d64d6f00..03ca45ce 100644 --- a/src/backup/snapshot_reader.rs +++ b/src/backup/snapshot_reader.rs @@ -11,10 +11,9 @@ use pbs_datastore::index::IndexFile; use pbs_datastore::fixed_index::FixedIndexReader; use pbs_datastore::dynamic_index::DynamicIndexReader; use pbs_datastore::manifest::{archive_type, ArchiveType, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME}; +use pbs_datastore::DataStore; use pbs_tools::fs::lock_dir_noblock_shared; -use crate::backup::DataStore; - /// Helper to access the contents of a datastore backup snapshot /// /// This make it easy to iterate over all used chunks and files. diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 3e3500a3..cdc48ed2 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -7,17 +7,14 @@ use std::time::Instant; use anyhow::{bail, format_err, Error}; use pbs_api_types::{Authid, CryptMode, VerifyState, UPID, SnapshotVerifyState}; -use pbs_datastore::{DataBlob, StoreProgress}; +use pbs_datastore::{DataStore, DataBlob, StoreProgress}; use pbs_datastore::backup_info::{BackupGroup, BackupDir, BackupInfo}; use pbs_datastore::index::IndexFile; use pbs_datastore::manifest::{archive_type, ArchiveType, BackupManifest, FileInfo}; use pbs_tools::fs::lock_dir_noblock_shared; use pbs_tools::{task_log, task::WorkerTaskContext}; -use crate::{ - backup::DataStore, - tools::ParallelHandler, -}; +use crate::tools::ParallelHandler; /// A VerifyWorker encapsulates a task worker, datastore and information about which chunks have /// already been verified or detected as corrupt. diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index ef4dd633..013c14cd 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -19,11 +19,11 @@ use proxmox::api::RpcEnvironmentType; use proxmox::sys::linux::socket::set_tcp_keepalive; use proxmox::tools::fs::CreateOptions; -use proxmox_rest_server::{rotate_task_log_archive, ApiConfig, RestServer, WorkerTask}; use pbs_tools::task_log; +use pbs_datastore::DataStore; +use proxmox_rest_server::{rotate_task_log_archive, ApiConfig, RestServer, WorkerTask}; use proxmox_backup::{ - backup::DataStore, server::{ auth::default_api_auth, jobstate::{ @@ -238,7 +238,7 @@ async fn run() -> Result<(), Error> { commando_sock.register_command( "datastore-removed".to_string(), |_value| { - if let Err(err) = proxmox_backup::backup::DataStore::remove_unused_datastores() { + if let Err(err) = DataStore::remove_unused_datastores() { log::error!("could not refresh datastores: {}", err); } Ok(Value::Null) diff --git a/src/server/gc_job.rs b/src/server/gc_job.rs index 183a00f1..794fe146 100644 --- a/src/server/gc_job.rs +++ b/src/server/gc_job.rs @@ -3,12 +3,10 @@ use anyhow::Error; use pbs_api_types::Authid; use pbs_tools::task_log; +use pbs_datastore::DataStore; use proxmox_rest_server::WorkerTask; -use crate::{ - server::jobstate::Job, - backup::DataStore, -}; +use crate::server::jobstate::Job; /// Runs a garbage collection job. pub fn do_garbage_collection_job( diff --git a/src/server/prune_job.rs b/src/server/prune_job.rs index 53740187..fc6443e9 100644 --- a/src/server/prune_job.rs +++ b/src/server/prune_job.rs @@ -4,15 +4,13 @@ use anyhow::Error; use pbs_datastore::backup_info::BackupInfo; use pbs_datastore::prune::compute_prune_info; +use pbs_datastore::DataStore; use pbs_api_types::{Authid, PRIV_DATASTORE_MODIFY, PruneOptions}; use pbs_config::CachedUserInfo; use pbs_tools::{task_log, task_warn}; use proxmox_rest_server::WorkerTask; -use crate::{ - backup::DataStore, - server::jobstate::Job, - }; +use crate::server::jobstate::Job; pub fn prune_datastore( worker: Arc, diff --git a/src/server/pull.rs b/src/server/pull.rs index cb85383b..61337038 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -13,7 +13,7 @@ use serde_json::json; use proxmox::api::error::{HttpError, StatusCode}; use pbs_api_types::{Authid, SnapshotListItem, GroupListItem}; -use pbs_datastore::{BackupInfo, BackupDir, BackupGroup, StoreProgress}; +use pbs_datastore::{DataStore, BackupInfo, BackupDir, BackupGroup, StoreProgress}; use pbs_datastore::data_blob::DataBlob; use pbs_datastore::dynamic_index::DynamicIndexReader; use pbs_datastore::fixed_index::FixedIndexReader; @@ -26,10 +26,7 @@ use pbs_tools::task_log; use pbs_client::{BackupReader, BackupRepository, HttpClient, HttpClientOptions, RemoteChunkReader}; use proxmox_rest_server::WorkerTask; -use crate::{ - backup::DataStore, - tools::ParallelHandler, -}; +use crate::tools::ParallelHandler; // fixme: implement filters // fixme: delete vanished groups diff --git a/src/server/verify_job.rs b/src/server/verify_job.rs index e90dcd4d..6aba97c9 100644 --- a/src/server/verify_job.rs +++ b/src/server/verify_job.rs @@ -3,11 +3,11 @@ use anyhow::{format_err, Error}; use pbs_tools::task_log; use pbs_api_types::{Authid, VerificationJobConfig}; use proxmox_rest_server::WorkerTask; +use pbs_datastore::DataStore; use crate::{ server::jobstate::Job, backup::{ - DataStore, verify_filter, verify_all_backups, }, diff --git a/src/tape/pool_writer/mod.rs b/src/tape/pool_writer/mod.rs index 2f7de2f5..b3f85a6e 100644 --- a/src/tape/pool_writer/mod.rs +++ b/src/tape/pool_writer/mod.rs @@ -19,10 +19,11 @@ use pbs_tape::{ TapeWrite, sg_tape::tape_alert_flags_critical, }; +use pbs_datastore::DataStore; use proxmox_rest_server::WorkerTask; use crate::{ - backup::{DataStore, SnapshotReader}, + backup::SnapshotReader, tape::{ TAPE_STATUS_DIR, MAX_CHUNK_ARCHIVE_SIZE, diff --git a/src/tape/pool_writer/new_chunks_iterator.rs b/src/tape/pool_writer/new_chunks_iterator.rs index 68fcc857..97203fea 100644 --- a/src/tape/pool_writer/new_chunks_iterator.rs +++ b/src/tape/pool_writer/new_chunks_iterator.rs @@ -3,9 +3,9 @@ use std::sync::{Arc, Mutex}; use anyhow::{format_err, Error}; -use pbs_datastore::DataBlob; +use pbs_datastore::{DataStore, DataBlob}; -use crate::backup::{DataStore, SnapshotReader}; +use crate::backup::SnapshotReader; use crate::tape::CatalogSet; /// Chunk iterator which use a separate thread to read chunks