cleanup proxmox_backup::backup module

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller 2021-08-30 11:49:22 +02:00
parent 97dfc62f0d
commit b2065dc7d2
33 changed files with 128 additions and 175 deletions

View File

@ -26,15 +26,37 @@ use proxmox::{http_err, identity, list_subdirs_api_method, sortable};
use pxar::accessor::aio::Accessor;
use pxar::EntryKind;
use pbs_api_types::{
Authid, BackupContent, Counts, CryptMode, DataStoreListItem, GarbageCollectionStatus,
GroupListItem, SnapshotListItem, SnapshotVerifyState, BACKUP_ARCHIVE_NAME_SCHEMA,
BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, DATASTORE_SCHEMA,
};
use pbs_client::pxar::create_zip;
use pbs_datastore::{BackupDir, BackupGroup, StoreProgress, CATALOG_NAME};
use pbs_datastore::backup_info::BackupInfo;
use pbs_datastore::cached_chunk_reader::CachedChunkReader;
use pbs_datastore::catalog::CatalogReader;
use pbs_datastore::data_blob::DataBlob;
use pbs_datastore::data_blob_reader::DataBlobReader;
use pbs_datastore::dynamic_index::{BufferedDynamicReader, DynamicIndexReader, LocalDynamicReadAt};
use pbs_datastore::fixed_index::{FixedIndexReader};
use pbs_datastore::index::IndexFile;
use pbs_datastore::manifest::{BackupManifest, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME};
use pbs_datastore::prune::{compute_prune_info, PruneOptions};
use pbs_tools::blocking::WrappedReaderStream;
use pbs_tools::stream::{AsyncReaderStream, AsyncChannelWriter};
use pbs_tools::json::{required_integer_param, required_string_param};
use crate::api2::types::*;
use crate::api2::types::{
ArchiveEntry, DataStoreStatus, RRDMode, RRDTimeFrameResolution,
IGNORE_VERIFIED_BACKUPS_SCHEMA, UPID_SCHEMA, VERIFICATION_OUTDATED_AFTER_SCHEMA
};
use crate::api2::node::rrd::create_value_from_rrd;
use crate::api2::helpers;
use crate::backup::*;
use crate::backup::{
check_backup_owner, verify_all_backups, verify_backup_group, verify_backup_dir, verify_filter,
DataStore, LocalChunkReader,
};
use crate::config::datastore;
use crate::config::cached_user_info::CachedUserInfo;

View File

@ -10,8 +10,13 @@ use proxmox::tools::digest_to_hex;
use proxmox::tools::fs::{replace_file, CreateOptions};
use proxmox::api::{RpcEnvironment, RpcEnvironmentType};
use pbs_datastore::DataBlob;
use pbs_datastore::backup_info::{BackupDir, BackupInfo};
use pbs_datastore::dynamic_index::DynamicIndexWriter;
use pbs_datastore::fixed_index::FixedIndexWriter;
use crate::api2::types::Authid;
use crate::backup::*;
use crate::backup::{verify_backup_dir_with_lock, DataStore};
use crate::server::WorkerTask;
use crate::server::formatter::*;
use hyper::{Body, Response};

View File

@ -15,9 +15,12 @@ use proxmox::api::schema::*;
use pbs_tools::fs::lock_dir_noblock_shared;
use pbs_tools::json::{required_array_param, required_integer_param, required_string_param};
use pbs_datastore::PROXMOX_BACKUP_PROTOCOL_ID_V1;
use pbs_datastore::backup_info::{BackupDir, BackupGroup, BackupInfo};
use pbs_datastore::index::IndexFile;
use pbs_datastore::manifest::{archive_type, ArchiveType};
use crate::server::{WorkerTask, H2Service};
use crate::backup::*;
use crate::backup::DataStore;
use crate::api2::types::*;
use crate::config::acl::PRIV_DATASTORE_BACKUP;
use crate::config::cached_user_info::CachedUserInfo;

View File

@ -12,10 +12,12 @@ use proxmox::{sortable, identity};
use proxmox::api::{ApiResponseFuture, ApiHandler, ApiMethod, RpcEnvironment};
use proxmox::api::schema::*;
use pbs_datastore::DataBlob;
use pbs_datastore::file_formats::{DataBlobHeader, EncryptedDataBlobHeader};
use pbs_tools::json::{required_integer_param, required_string_param};
use crate::api2::types::*;
use crate::backup::*;
use crate::backup::DataStore;
use super::environment::*;

View File

@ -7,6 +7,8 @@ use ::serde::{Deserialize, Serialize};
use proxmox::api::{api, Router, RpcEnvironment, Permission};
use proxmox::api::section_config::SectionConfigData;
use proxmox::api::schema::{ApiType, parse_property_string};
use pbs_datastore::chunk_store::ChunkStore;
use pbs_datastore::task::TaskState;
use crate::api2::config::sync::delete_sync_job;
@ -17,7 +19,7 @@ use crate::api2::admin::{
verify::list_verification_jobs,
};
use crate::api2::types::*;
use crate::backup::*;
use crate::backup::BackupLockGuard;
use crate::config::cached_user_info::CachedUserInfo;
use crate::config::datastore::{self, DataStoreConfig, DataStoreConfigUpdater};
use crate::config::acl::{PRIV_DATASTORE_ALLOCATE, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY};

View File

@ -11,7 +11,9 @@ use proxmox::{
},
};
use pbs_api_types::Fingerprint;
use pbs_datastore::{KeyInfo, Kdf};
use pbs_datastore::key_derivation::KeyConfig;
use crate::{
config::{
@ -33,11 +35,7 @@ use crate::{
PROXMOX_CONFIG_DIGEST_SCHEMA,
PASSWORD_HINT_SCHEMA,
},
backup::{
open_backup_lockfile,
KeyConfig,
Fingerprint,
},
backup::open_backup_lockfile,
};
#[api(

View File

@ -7,8 +7,9 @@ use hyper::{Body, Response, StatusCode, header};
use proxmox::http_bail;
use pbs_datastore::catalog::{CatalogReader, DirEntryAttribute};
use crate::api2::types::ArchiveEntry;
use crate::backup::{CatalogReader, DirEntryAttribute};
pub async fn create_download_response(path: PathBuf) -> Result<Response<Body>, Error> {
let file = match tokio::fs::File::open(path.clone()).await {

View File

@ -5,8 +5,10 @@ use serde_json::{json, Value};
use proxmox::api::{RpcEnvironment, RpcEnvironmentType};
use pbs_datastore::backup_info::BackupDir;
use crate::api2::types::Authid;
use crate::backup::*;
use crate::backup::DataStore;
use crate::server::formatter::*;
use crate::server::WorkerTask;

View File

@ -30,6 +30,9 @@ use proxmox::{
use pbs_tools::fs::lock_dir_noblock_shared;
use pbs_tools::json::{required_integer_param, required_string_param};
use pbs_datastore::PROXMOX_BACKUP_READER_PROTOCOL_ID_V1;
use pbs_datastore::backup_info::BackupDir;
use pbs_datastore::index::IndexFile;
use pbs_datastore::manifest::{archive_type, ArchiveType};
use crate::{
api2::{
@ -43,13 +46,7 @@ use crate::{
Authid,
},
},
backup::{
DataStore,
ArchiveType,
BackupDir,
IndexFile,
archive_type,
},
backup::DataStore,
server::{
WorkerTask,
H2Service,

View File

@ -15,7 +15,9 @@ use proxmox::{
},
};
use pbs_datastore::{task_log, task_warn};
use pbs_api_types::{Authid, Userid};
use pbs_datastore::{task_log, task_warn, StoreProgress};
use pbs_datastore::backup_info::{BackupDir, BackupInfo};
use pbs_datastore::task::TaskState;
use crate::{
@ -42,18 +44,11 @@ use crate::{
compute_schedule_status,
},
},
backup::{
DataStore,
BackupDir,
BackupInfo,
StoreProgress,
},
backup::DataStore,
api2::types::{
Authid,
UPID_SCHEMA,
JOB_ID_SCHEMA,
MediaPoolConfig,
Userid,
},
server::WorkerTask,
tape::{

View File

@ -10,6 +10,8 @@ use proxmox::{
tools::Uuid,
};
use pbs_datastore::backup_info::BackupDir;
use crate::{
config::{
self,
@ -34,9 +36,6 @@ use crate::{
MediaContentEntry,
VAULT_NAME_SCHEMA,
},
backup::{
BackupDir,
},
tape::{
TAPE_STATUS_DIR,
Inventory,

View File

@ -28,7 +28,13 @@ use proxmox::{
},
};
use pbs_datastore::{task_log, task_warn};
use pbs_api_types::CryptMode;
use pbs_datastore::{task_log, task_warn, DataBlob};
use pbs_datastore::backup_info::BackupDir;
use pbs_datastore::dynamic_index::DynamicIndexReader;
use pbs_datastore::fixed_index::FixedIndexReader;
use pbs_datastore::index::IndexFile;
use pbs_datastore::manifest::{archive_type, ArchiveType, BackupManifest, MANIFEST_BLOB_NAME};
use pbs_datastore::task::TaskState;
use crate::{
@ -51,19 +57,7 @@ use crate::{
PRIV_TAPE_READ,
},
},
backup::{
ArchiveType,
archive_type,
IndexFile,
MANIFEST_BLOB_NAME,
CryptMode,
DataStore,
DynamicIndexReader,
FixedIndexReader,
BackupDir,
DataBlob,
BackupManifest,
},
backup::DataStore,
server::{
lookup_user_email,
WorkerTask,

View File

@ -6,12 +6,9 @@ use serde::{Deserialize, Serialize};
use proxmox::api::{api, schema::*};
use proxmox::const_regex;
use pbs_datastore::catalog::CatalogEntryType;
use pbs_datastore::catalog::{CatalogEntryType, DirEntryAttribute};
use crate::{
backup::DirEntryAttribute,
config::acl::Role,
};
use crate::config::acl::Role;
mod tape;
pub use tape::*;

View File

@ -36,52 +36,6 @@ pub fn backup_group() -> Result<nix::unistd::Group, Error> {
}
}
pub use pbs_datastore::backup_info;
pub use pbs_datastore::backup_info::*;
pub use pbs_datastore::catalog;
pub use pbs_datastore::catalog::*;
pub use pbs_datastore::checksum_reader;
pub use pbs_datastore::checksum_reader::*;
pub use pbs_datastore::checksum_writer;
pub use pbs_datastore::checksum_writer::*;
pub use pbs_datastore::chunk_stat;
pub use pbs_datastore::chunk_stat::*;
pub use pbs_datastore::chunk_store;
pub use pbs_datastore::chunk_store::*;
pub use pbs_datastore::chunker;
pub use pbs_datastore::chunker::*;
pub use pbs_datastore::crypt_config;
pub use pbs_datastore::crypt_config::*;
pub use pbs_datastore::crypt_reader;
pub use pbs_datastore::crypt_reader::*;
pub use pbs_datastore::crypt_writer;
pub use pbs_datastore::crypt_writer::*;
pub use pbs_datastore::data_blob;
pub use pbs_datastore::data_blob::*;
pub use pbs_datastore::data_blob_reader;
pub use pbs_datastore::data_blob_reader::*;
pub use pbs_datastore::data_blob_writer;
pub use pbs_datastore::data_blob_writer::*;
pub use pbs_datastore::file_formats;
pub use pbs_datastore::file_formats::*;
pub use pbs_datastore::index;
pub use pbs_datastore::index::*;
pub use pbs_datastore::key_derivation;
pub use pbs_datastore::key_derivation::*;
pub use pbs_datastore::manifest;
pub use pbs_datastore::manifest::*;
pub use pbs_datastore::prune;
pub use pbs_datastore::prune::*;
pub use pbs_datastore::store_progress::StoreProgress;
pub use pbs_datastore::cached_chunk_reader::*;
pub use pbs_datastore::dynamic_index::*;
pub use pbs_datastore::fixed_index;
pub use pbs_datastore::fixed_index::*;
pub use pbs_datastore::read_chunk::*;
// Split
mod read_chunk;
pub use read_chunk::*;

View File

@ -6,26 +6,17 @@ use std::time::Instant;
use anyhow::{bail, format_err, Error};
use pbs_datastore::task_log;
use pbs_api_types::CryptMode;
use pbs_datastore::{task_log, DataBlob, StoreProgress};
use pbs_datastore::backup_info::{BackupGroup, BackupDir, BackupInfo};
use pbs_datastore::index::IndexFile;
use pbs_datastore::manifest::{archive_type, ArchiveType, BackupManifest, FileInfo};
use pbs_datastore::task::TaskState;
use pbs_tools::fs::lock_dir_noblock_shared;
use crate::{
api2::types::*,
backup::{
DataStore,
StoreProgress,
DataBlob,
BackupGroup,
BackupDir,
BackupInfo,
BackupManifest,
IndexFile,
CryptMode,
FileInfo,
ArchiveType,
archive_type,
},
backup::DataStore,
server::UPID,
tools::ParallelHandler,
};

View File

@ -462,10 +462,8 @@ async fn schedule_datastore_garbage_collection() {
async fn schedule_datastore_prune() {
use pbs_datastore::prune::PruneOptions;
use proxmox_backup::{
backup::{
PruneOptions,
},
config::datastore::{
self,
DataStoreConfig,

View File

@ -16,7 +16,13 @@ use proxmox::api::{
use pxar::accessor::aio::Accessor;
use pxar::decoder::aio::Decoder;
use pbs_api_types::CryptMode;
use pbs_datastore::{CryptConfig, CATALOG_NAME};
use pbs_datastore::backup_info::BackupDir;
use pbs_datastore::catalog::{CatalogReader, DirEntryAttribute};
use pbs_datastore::dynamic_index::{BufferedDynamicReader, LocalDynamicReadAt};
use pbs_datastore::index::IndexFile;
use pbs_datastore::key_derivation::decrypt_key;
use pbs_client::{BackupReader, RemoteChunkReader};
use pbs_client::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq};
use pbs_client::tools::{
@ -29,10 +35,6 @@ use pbs_client::tools::{
};
use proxmox_backup::api2::{helpers, types::ArchiveEntry};
use proxmox_backup::backup::{
decrypt_key, BackupDir, BufferedDynamicReader, CatalogReader, CryptConfig, CryptMode,
DirEntryAttribute, LocalDynamicReadAt, CATALOG_NAME,
};
use proxmox_backup::tools;
mod proxmox_file_restore;

View File

@ -11,8 +11,9 @@ use serde_json::{json, Value};
use proxmox::api::{api, cli::*};
use pbs_client::BackupRepository;
use pbs_datastore::backup_info::BackupDir;
use pbs_datastore::manifest::BackupManifest;
use proxmox_backup::backup::{BackupDir, BackupManifest};
use proxmox_backup::api2::types::ArchiveEntry;
use super::block_driver_qemu::QemuBlockDriver;

View File

@ -11,9 +11,9 @@ use serde_json::json;
use proxmox::tools::fs::lock_file;
use pbs_client::{DEFAULT_VSOCK_PORT, BackupRepository, VsockClient};
use pbs_datastore::backup_info::BackupDir;
use proxmox_backup::api2::types::ArchiveEntry;
use proxmox_backup::backup::BackupDir;
use proxmox_backup::tools;
use super::block_driver::*;

View File

@ -20,12 +20,12 @@ use proxmox::api::{
use proxmox::{identity, list_subdirs_api_method, sortable};
use pbs_client::pxar::{create_archive, Flags, PxarCreateOptions, ENCODER_MAX_ENTRIES};
use pbs_datastore::catalog::DirEntryAttribute;
use pbs_tools::fs::read_subdir;
use pbs_tools::json::required_string_param;
use pbs_tools::zip::zip_directory;
use proxmox_backup::api2::types::*;
use proxmox_backup::backup::DirEntryAttribute;
use pxar::encoder::aio::TokioWriter;

View File

@ -11,6 +11,7 @@ use proxmox::{
sys::linux::tty,
};
use pbs_api_types::Fingerprint;
use pbs_datastore::Kdf;
use pbs_datastore::paperkey::{PaperkeyFormat, generate_paper_key};
@ -24,7 +25,6 @@ use proxmox_backup::{
PASSWORD_HINT_SCHEMA,
},
},
backup::Fingerprint,
config::tape_encryption_keys::{
load_key_configs,
complete_key_fingerprint,

View File

@ -18,9 +18,10 @@ use proxmox::{
tools::Uuid,
};
use pbs_api_types::Fingerprint;
use proxmox_backup::{
config,
backup::Fingerprint,
api2::types::{
LTO_DRIVE_PATH_SCHEMA,
DRIVE_NAME_SCHEMA,

View File

@ -16,14 +16,10 @@ use anyhow::{bail, Error};
use serde::{Deserialize, Serialize};
use proxmox::tools::fs::file_read_optional_string;
use pbs_api_types::Fingerprint;
use pbs_datastore::key_derivation::KeyConfig;
use crate::{
backup::{
open_backup_lockfile,
Fingerprint,
KeyConfig,
},
};
use crate::backup::open_backup_lockfile;
mod hex_key {
use serde::{self, Deserialize, Serializer, Deserializer};

View File

@ -3,12 +3,14 @@ use std::sync::Arc;
use anyhow::Error;
use pbs_datastore::{task_log, task_warn};
use pbs_datastore::backup_info::BackupInfo;
use pbs_datastore::prune::{compute_prune_info, PruneOptions};
use crate::{
api2::types::*,
config::acl::PRIV_DATASTORE_MODIFY,
config::cached_user_info::CachedUserInfo,
backup::{compute_prune_info, BackupInfo, DataStore, PruneOptions},
backup::DataStore,
server::jobstate::Job,
server::WorkerTask,
};

View File

@ -27,13 +27,12 @@ use proxmox::{
sys::error::SysResult,
};
use pbs_api_types::Fingerprint;
use pbs_datastore::key_derivation::KeyConfig;
use crate::{
config,
tools::run_command,
backup::{
Fingerprint,
KeyConfig,
},
api2::types::{
MamAttribute,
LtoDriveAndMediaStatus,

View File

@ -28,14 +28,12 @@ use proxmox::{
api::section_config::SectionConfigData,
};
use pbs_datastore::task_log;
use pbs_api_types::Fingerprint;
use pbs_datastore::key_derivation::KeyConfig;
use pbs_datastore::task::TaskState;
use pbs_datastore::task_log;
use crate::{
backup::{
Fingerprint,
KeyConfig,
},
api2::types::{
VirtualTapeDrive,
LtoTapeDrive,

View File

@ -10,8 +10,9 @@ use proxmox::tools::{
fs::{replace_file, CreateOptions},
};
use pbs_datastore::key_derivation::KeyConfig;
use crate::{
backup::KeyConfig,
tape::{
TapeWrite,
TapeRead,

View File

@ -8,7 +8,7 @@ use proxmox::tools::{
io::ReadExt,
};
use crate::backup::DataBlob;
use pbs_datastore::DataBlob;
use crate::tape::{
TapeWrite,

View File

@ -1,6 +1,17 @@
//! File format definitions and implementations for data written to
//! tapes
use std::collections::HashMap;
use anyhow::{bail, Error};
use bitflags::bitflags;
use endian_trait::Endian;
use serde::{Deserialize, Serialize};
use proxmox::tools::Uuid;
use pbs_api_types::Fingerprint;
mod blocked_reader;
pub use blocked_reader::*;
@ -22,17 +33,6 @@ pub use multi_volume_writer::*;
mod multi_volume_reader;
pub use multi_volume_reader::*;
use std::collections::HashMap;
use anyhow::{bail, Error};
use ::serde::{Deserialize, Serialize};
use endian_trait::Endian;
use bitflags::bitflags;
use proxmox::tools::Uuid;
use crate::backup::Fingerprint;
/// We use 256KB blocksize (always)
pub const PROXMOX_TAPE_BLOCK_SIZE: usize = 256*1024;

View File

@ -6,21 +6,14 @@ use std::fs::File;
use anyhow::{bail, Error};
use nix::dir::Dir;
use pbs_datastore::backup_info::BackupDir;
use pbs_datastore::index::IndexFile;
use pbs_datastore::fixed_index::FixedIndexReader;
use pbs_datastore::dynamic_index::DynamicIndexReader;
use pbs_datastore::manifest::{archive_type, ArchiveType, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME};
use pbs_tools::fs::lock_dir_noblock_shared;
use crate::{
backup::{
DataStore,
BackupDir,
ArchiveType,
IndexFile,
FixedIndexReader,
DynamicIndexReader,
MANIFEST_BLOB_NAME,
CLIENT_LOG_BLOB_NAME,
archive_type,
},
};
use crate::backup::DataStore;
/// Helper to access the contents of a datastore backup snapshot
///

View File

@ -9,6 +9,7 @@ use anyhow::{bail, format_err, Error};
use endian_trait::Endian;
use pbs_tools::fs::read_subdir;
use pbs_datastore::backup_info::BackupDir;
use proxmox::tools::{
Uuid,
@ -24,7 +25,6 @@ use proxmox::tools::{
};
use crate::{
backup::BackupDir,
tape::{
MediaId,
file_formats::MediaSetLabel,

View File

@ -10,12 +10,14 @@
use std::path::{PathBuf, Path};
use anyhow::{bail, Error};
use ::serde::{Deserialize, Serialize};
use serde::{Deserialize, Serialize};
use proxmox::tools::Uuid;
use pbs_api_types::Fingerprint;
use crate::{
backup::{Fingerprint, BackupLockGuard},
backup::BackupLockGuard,
api2::types::{
MediaStatus,
MediaLocation,

View File

@ -3,15 +3,13 @@ use std::sync::{Arc, Mutex};
use anyhow::{format_err, Error};
use pbs_datastore::DataBlob;
use crate::{
backup::{
DataStore,
DataBlob,
},
backup::DataStore,
tape::{
CatalogSet,
SnapshotReader,
},
};