diff --git a/pbs-api-types/Cargo.toml b/pbs-api-types/Cargo.toml index cd3a7073..2463d69d 100644 --- a/pbs-api-types/Cargo.toml +++ b/pbs-api-types/Cargo.toml @@ -16,3 +16,4 @@ serde = { version = "1.0", features = ["derive"] } proxmox = { version = "0.11.5", default-features = false, features = [ "api-macro" ] } pbs-systemd = { path = "../pbs-systemd" } +pbs-tools = { path = "../pbs-tools" } diff --git a/pbs-api-types/src/crypto.rs b/pbs-api-types/src/crypto.rs new file mode 100644 index 00000000..7b36e85f --- /dev/null +++ b/pbs-api-types/src/crypto.rs @@ -0,0 +1,57 @@ +use std::fmt::{self, Display}; + +use anyhow::Error; +use serde::{Deserialize, Serialize}; + +use proxmox::api::api; + +use pbs_tools::format::{as_fingerprint, bytes_as_fingerprint}; + +#[api(default: "encrypt")] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +/// Defines whether data is encrypted (using an AEAD cipher), only signed, or neither. +pub enum CryptMode { + /// Don't encrypt. + None, + /// Encrypt. + Encrypt, + /// Only sign. + SignOnly, +} + +#[derive(Debug, Eq, PartialEq, Hash, Clone, Deserialize, Serialize)] +#[serde(transparent)] +/// 32-byte fingerprint, usually calculated with SHA256. +pub struct Fingerprint { + #[serde(with = "bytes_as_fingerprint")] + bytes: [u8; 32], +} + +impl Fingerprint { + pub fn new(bytes: [u8; 32]) -> Self { + Self { bytes } + } + pub fn bytes(&self) -> &[u8; 32] { + &self.bytes + } +} + +/// Display as short key ID +impl Display for Fingerprint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", as_fingerprint(&self.bytes[0..8])) + } +} + +impl std::str::FromStr for Fingerprint { + type Err = Error; + + fn from_str(s: &str) -> Result { + let mut tmp = s.to_string(); + tmp.retain(|c| c != ':'); + let bytes = proxmox::tools::hex_to_digest(&tmp)?; + Ok(Fingerprint::new(bytes)) + } +} + diff --git a/pbs-api-types/src/lib.rs b/pbs-api-types/src/lib.rs index 8e2d1a7c..2d15e92e 100644 --- a/pbs-api-types/src/lib.rs +++ b/pbs-api-types/src/lib.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use proxmox::api::api; -use proxmox::api::schema::{ApiStringFormat, Schema, StringSchema}; +use proxmox::api::schema::{ApiStringFormat, EnumEntry, IntegerSchema, Schema, StringSchema}; use proxmox::const_regex; use proxmox::{IPRE, IPRE_BRACKET, IPV4OCTET, IPV4RE, IPV6H16, IPV6LS32, IPV6RE}; @@ -43,6 +43,9 @@ pub use userid::{PROXMOX_GROUP_ID_SCHEMA, PROXMOX_TOKEN_ID_SCHEMA, PROXMOX_TOKEN pub mod upid; pub use upid::UPID; +mod crypto; +pub use crypto::{CryptMode, Fingerprint}; + #[rustfmt::skip] #[macro_use] mod local_macros { @@ -115,6 +118,26 @@ pub const CIDR_V4_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&CIDR_V4_RE pub const CIDR_V6_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&CIDR_V6_REGEX); pub const CIDR_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&CIDR_REGEX); +pub const BACKUP_ID_SCHEMA: Schema = StringSchema::new("Backup ID.") + .format(&BACKUP_ID_FORMAT) + .schema(); +pub const BACKUP_TYPE_SCHEMA: Schema = StringSchema::new("Backup type.") + .format(&ApiStringFormat::Enum(&[ + EnumEntry::new("vm", "Virtual Machine Backup"), + EnumEntry::new("ct", "Container Backup"), + EnumEntry::new("host", "Host Backup"), + ])) + .schema(); +pub const BACKUP_TIME_SCHEMA: Schema = IntegerSchema::new("Backup time (Unix epoch.)") + .minimum(1_547_797_308) + .schema(); + +pub const DATASTORE_SCHEMA: Schema = StringSchema::new("Datastore name.") + .format(&PROXMOX_SAFE_ID_FORMAT) + .min_length(3) + .max_length(32) + .schema(); + pub const FINGERPRINT_SHA256_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&FINGERPRINT_SHA256_REGEX); @@ -197,3 +220,207 @@ pub const CHUNK_DIGEST_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&SHA25 pub const PASSWORD_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&PASSWORD_REGEX); pub const UUID_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&UUID_REGEX); + +pub const BACKUP_ARCHIVE_NAME_SCHEMA: Schema = StringSchema::new("Backup archive name.") + .format(&PROXMOX_SAFE_ID_FORMAT) + .schema(); + +// Complex type definitions + +#[api( + properties: { + "filename": { + schema: BACKUP_ARCHIVE_NAME_SCHEMA, + }, + "crypt-mode": { + type: CryptMode, + optional: true, + }, + }, +)] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +/// Basic information about archive files inside a backup snapshot. +pub struct BackupContent { + pub filename: String, + /// Info if file is encrypted, signed, or neither. + #[serde(skip_serializing_if = "Option::is_none")] + pub crypt_mode: Option, + /// Archive size (from backup manifest). + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, +} + +#[api()] +#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +/// Result of a verify operation. +pub enum VerifyState { + /// Verification was successful + Ok, + /// Verification reported one or more errors + Failed, +} + +#[api( + properties: { + upid: { + type: UPID, + }, + state: { + type: VerifyState, + }, + }, +)] +#[derive(Serialize, Deserialize)] +/// Task properties. +pub struct SnapshotVerifyState { + /// UPID of the verify task + pub upid: UPID, + /// State of the verification. Enum. + pub state: VerifyState, +} + +#[api( + properties: { + "backup-type": { + schema: BACKUP_TYPE_SCHEMA, + }, + "backup-id": { + schema: BACKUP_ID_SCHEMA, + }, + "backup-time": { + schema: BACKUP_TIME_SCHEMA, + }, + comment: { + schema: SINGLE_LINE_COMMENT_SCHEMA, + optional: true, + }, + verification: { + type: SnapshotVerifyState, + optional: true, + }, + fingerprint: { + type: String, + optional: true, + }, + files: { + items: { + schema: BACKUP_ARCHIVE_NAME_SCHEMA + }, + }, + owner: { + type: Authid, + optional: true, + }, + }, +)] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +/// Basic information about backup snapshot. +pub struct SnapshotListItem { + pub backup_type: String, // enum + pub backup_id: String, + pub backup_time: i64, + /// The first line from manifest "notes" + #[serde(skip_serializing_if = "Option::is_none")] + pub comment: Option, + /// The result of the last run verify task + #[serde(skip_serializing_if = "Option::is_none")] + pub verification: Option, + /// Fingerprint of encryption key + #[serde(skip_serializing_if = "Option::is_none")] + pub fingerprint: Option, + /// List of contained archive files. + pub files: Vec, + /// Overall snapshot size (sum of all archive sizes). + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + /// The owner of the snapshots group + #[serde(skip_serializing_if = "Option::is_none")] + pub owner: Option, +} + +#[api( + properties: { + "backup-type": { + schema: BACKUP_TYPE_SCHEMA, + }, + "backup-id": { + schema: BACKUP_ID_SCHEMA, + }, + "last-backup": { + schema: BACKUP_TIME_SCHEMA, + }, + "backup-count": { + type: Integer, + }, + files: { + items: { + schema: BACKUP_ARCHIVE_NAME_SCHEMA + }, + }, + owner: { + type: Authid, + optional: true, + }, + }, +)] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +/// Basic information about a backup group. +pub struct GroupListItem { + pub backup_type: String, // enum + pub backup_id: String, + pub last_backup: i64, + /// Number of contained snapshots + pub backup_count: u64, + /// List of contained archive files. + pub files: Vec, + /// The owner of group + #[serde(skip_serializing_if = "Option::is_none")] + pub owner: Option, +} + +#[api( + properties: { + store: { + schema: DATASTORE_SCHEMA, + }, + comment: { + optional: true, + schema: SINGLE_LINE_COMMENT_SCHEMA, + }, + }, +)] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +/// Basic information about a datastore. +pub struct DataStoreListItem { + pub store: String, + pub comment: Option, +} + +#[api( + properties: { + "backup-type": { + schema: BACKUP_TYPE_SCHEMA, + }, + "backup-id": { + schema: BACKUP_ID_SCHEMA, + }, + "backup-time": { + schema: BACKUP_TIME_SCHEMA, + }, + }, +)] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +/// Prune result. +pub struct PruneListItem { + pub backup_type: String, // enum + pub backup_id: String, + pub backup_time: i64, + /// Keep snapshot + pub keep: bool, +} diff --git a/pbs-datastore/src/crypt_config.rs b/pbs-datastore/src/crypt_config.rs index 8d681d58..a2f51302 100644 --- a/pbs-datastore/src/crypt_config.rs +++ b/pbs-datastore/src/crypt_config.rs @@ -7,19 +7,14 @@ //! encryption](https://en.wikipedia.org/wiki/Authenticated_encryption) //! for a short introduction. -use std::fmt; -use std::fmt::Display; use std::io::Write; use anyhow::{Error}; use openssl::hash::MessageDigest; use openssl::pkcs5::pbkdf2_hmac; use openssl::symm::{decrypt_aead, Cipher, Crypter, Mode}; -use serde::{Deserialize, Serialize}; -use proxmox::api::api; - -use pbs_tools::format::{as_fingerprint, bytes_as_fingerprint}; +pub use pbs_api_types::{CryptMode, Fingerprint}; // openssl::sha::sha256(b"Proxmox Backup Encryption Key Fingerprint") /// This constant is used to compute fingerprints. @@ -29,53 +24,6 @@ const FINGERPRINT_INPUT: [u8; 32] = [ 97, 64, 127, 19, 76, 114, 93, 223, 48, 153, 45, 37, 236, 69, 237, 38, ]; -#[api(default: "encrypt")] -#[derive(Copy, Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -/// Defines whether data is encrypted (using an AEAD cipher), only signed, or neither. -pub enum CryptMode { - /// Don't encrypt. - None, - /// Encrypt. - Encrypt, - /// Only sign. - SignOnly, -} - -#[derive(Debug, Eq, PartialEq, Hash, Clone, Deserialize, Serialize)] -#[serde(transparent)] -/// 32-byte fingerprint, usually calculated with SHA256. -pub struct Fingerprint { - #[serde(with = "bytes_as_fingerprint")] - bytes: [u8; 32], -} - -impl Fingerprint { - pub fn new(bytes: [u8; 32]) -> Self { - Self { bytes } - } - pub fn bytes(&self) -> &[u8; 32] { - &self.bytes - } -} - -/// Display as short key ID -impl Display for Fingerprint { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", as_fingerprint(&self.bytes[0..8])) - } -} - -impl std::str::FromStr for Fingerprint { - type Err = Error; - - fn from_str(s: &str) -> Result { - let mut tmp = s.to_string(); - tmp.retain(|c| c != ':'); - let bytes = proxmox::tools::hex_to_digest(&tmp)?; - Ok(Fingerprint::new(bytes)) - } -} /// Encryption Configuration with secret key /// diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs index f0b9f57d..3034ec4e 100644 --- a/pbs-datastore/src/lib.rs +++ b/pbs-datastore/src/lib.rs @@ -197,6 +197,7 @@ pub mod key_derivation; pub mod manifest; pub mod prune; pub mod read_chunk; +pub mod store_progress; pub mod task; pub mod dynamic_index; @@ -216,3 +217,4 @@ pub use key_derivation::{ }; pub use key_derivation::{Kdf, KeyConfig, KeyDerivationConfig, KeyInfo}; pub use manifest::BackupManifest; +pub use store_progress::StoreProgress; diff --git a/pbs-datastore/src/manifest.rs b/pbs-datastore/src/manifest.rs index 1f8d49f4..7799f906 100644 --- a/pbs-datastore/src/manifest.rs +++ b/pbs-datastore/src/manifest.rs @@ -85,21 +85,26 @@ pub enum ArchiveType { Blob, } +impl ArchiveType { + pub fn from_path(archive_name: impl AsRef) -> Result { + let archive_name = archive_name.as_ref(); + let archive_type = match archive_name.extension().and_then(|ext| ext.to_str()) { + Some("didx") => ArchiveType::DynamicIndex, + Some("fidx") => ArchiveType::FixedIndex, + Some("blob") => ArchiveType::Blob, + _ => bail!("unknown archive type: {:?}", archive_name), + }; + Ok(archive_type) + } +} + +//#[deprecated(note = "use ArchivType::from_path instead")] later... pub fn archive_type>( archive_name: P, ) -> Result { - - let archive_name = archive_name.as_ref(); - let archive_type = match archive_name.extension().and_then(|ext| ext.to_str()) { - Some("didx") => ArchiveType::DynamicIndex, - Some("fidx") => ArchiveType::FixedIndex, - Some("blob") => ArchiveType::Blob, - _ => bail!("unknown archive type: {:?}", archive_name), - }; - Ok(archive_type) + ArchiveType::from_path(archive_name) } - impl BackupManifest { pub fn new(snapshot: BackupDir) -> Self { @@ -114,7 +119,7 @@ impl BackupManifest { } pub fn add_file(&mut self, filename: String, size: u64, csum: [u8; 32], crypt_mode: CryptMode) -> Result<(), Error> { - let _archive_type = archive_type(&filename)?; // check type + let _archive_type = ArchiveType::from_path(&filename)?; // check type self.files.push(FileInfo { filename, size, csum, crypt_mode }); Ok(()) } diff --git a/src/backup/store_progress.rs b/pbs-datastore/src/store_progress.rs similarity index 100% rename from src/backup/store_progress.rs rename to pbs-datastore/src/store_progress.rs diff --git a/src/api2/types/mod.rs b/src/api2/types/mod.rs index 287abc7e..d3c16b96 100644 --- a/src/api2/types/mod.rs +++ b/src/api2/types/mod.rs @@ -9,12 +9,7 @@ use proxmox::const_regex; use pbs_datastore::catalog::CatalogEntryType; use crate::{ - backup::{ - CryptMode, - Fingerprint, - DirEntryAttribute, - }, - server::UPID, + backup::DirEntryAttribute, config::acl::Role, }; @@ -244,39 +239,10 @@ pub struct AclListItem { pub roleid: String, } -pub const BACKUP_ARCHIVE_NAME_SCHEMA: Schema = - StringSchema::new("Backup archive name.") - .format(&PROXMOX_SAFE_ID_FORMAT) - .schema(); - -pub const BACKUP_TYPE_SCHEMA: Schema = - StringSchema::new("Backup type.") - .format(&ApiStringFormat::Enum(&[ - EnumEntry::new("vm", "Virtual Machine Backup"), - EnumEntry::new("ct", "Container Backup"), - EnumEntry::new("host", "Host Backup")])) - .schema(); - -pub const BACKUP_ID_SCHEMA: Schema = - StringSchema::new("Backup ID.") - .format(&BACKUP_ID_FORMAT) - .schema(); - -pub const BACKUP_TIME_SCHEMA: Schema = - IntegerSchema::new("Backup time (Unix epoch.)") - .minimum(1_547_797_308) - .schema(); - pub const UPID_SCHEMA: Schema = StringSchema::new("Unique Process/Task ID.") .max_length(256) .schema(); -pub const DATASTORE_SCHEMA: Schema = StringSchema::new("Datastore name.") - .format(&PROXMOX_SAFE_ID_FORMAT) - .min_length(3) - .max_length(32) - .schema(); - pub const DATASTORE_MAP_SCHEMA: Schema = StringSchema::new("Datastore mapping.") .format(&DATASTORE_MAP_FORMAT) .min_length(3) @@ -391,180 +357,6 @@ pub const REALM_ID_SCHEMA: Schema = StringSchema::new("Realm name.") // Complex type definitions -#[api( - properties: { - store: { - schema: DATASTORE_SCHEMA, - }, - comment: { - optional: true, - schema: SINGLE_LINE_COMMENT_SCHEMA, - }, - }, -)] -#[derive(Serialize, Deserialize)] -#[serde(rename_all="kebab-case")] -/// Basic information about a datastore. -pub struct DataStoreListItem { - pub store: String, - pub comment: Option, -} - -#[api( - properties: { - "backup-type": { - schema: BACKUP_TYPE_SCHEMA, - }, - "backup-id": { - schema: BACKUP_ID_SCHEMA, - }, - "last-backup": { - schema: BACKUP_TIME_SCHEMA, - }, - "backup-count": { - type: Integer, - }, - files: { - items: { - schema: BACKUP_ARCHIVE_NAME_SCHEMA - }, - }, - owner: { - type: Authid, - optional: true, - }, - }, -)] -#[derive(Serialize, Deserialize)] -#[serde(rename_all="kebab-case")] -/// Basic information about a backup group. -pub struct GroupListItem { - pub backup_type: String, // enum - pub backup_id: String, - pub last_backup: i64, - /// Number of contained snapshots - pub backup_count: u64, - /// List of contained archive files. - pub files: Vec, - /// The owner of group - #[serde(skip_serializing_if="Option::is_none")] - pub owner: Option, -} - -#[api()] -#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -/// Result of a verify operation. -pub enum VerifyState { - /// Verification was successful - Ok, - /// Verification reported one or more errors - Failed, -} - -#[api( - properties: { - upid: { - schema: UPID_SCHEMA - }, - state: { - type: VerifyState - }, - }, -)] -#[derive(Serialize, Deserialize)] -/// Task properties. -pub struct SnapshotVerifyState { - /// UPID of the verify task - pub upid: UPID, - /// State of the verification. Enum. - pub state: VerifyState, -} - -#[api( - properties: { - "backup-type": { - schema: BACKUP_TYPE_SCHEMA, - }, - "backup-id": { - schema: BACKUP_ID_SCHEMA, - }, - "backup-time": { - schema: BACKUP_TIME_SCHEMA, - }, - comment: { - schema: SINGLE_LINE_COMMENT_SCHEMA, - optional: true, - }, - verification: { - type: SnapshotVerifyState, - optional: true, - }, - fingerprint: { - type: String, - optional: true, - }, - files: { - items: { - schema: BACKUP_ARCHIVE_NAME_SCHEMA - }, - }, - owner: { - type: Authid, - optional: true, - }, - }, -)] -#[derive(Serialize, Deserialize)] -#[serde(rename_all="kebab-case")] -/// Basic information about backup snapshot. -pub struct SnapshotListItem { - pub backup_type: String, // enum - pub backup_id: String, - pub backup_time: i64, - /// The first line from manifest "notes" - #[serde(skip_serializing_if="Option::is_none")] - pub comment: Option, - /// The result of the last run verify task - #[serde(skip_serializing_if="Option::is_none")] - pub verification: Option, - /// Fingerprint of encryption key - #[serde(skip_serializing_if="Option::is_none")] - pub fingerprint: Option, - /// List of contained archive files. - pub files: Vec, - /// Overall snapshot size (sum of all archive sizes). - #[serde(skip_serializing_if="Option::is_none")] - pub size: Option, - /// The owner of the snapshots group - #[serde(skip_serializing_if="Option::is_none")] - pub owner: Option, -} - -#[api( - properties: { - "backup-type": { - schema: BACKUP_TYPE_SCHEMA, - }, - "backup-id": { - schema: BACKUP_ID_SCHEMA, - }, - "backup-time": { - schema: BACKUP_TIME_SCHEMA, - }, - }, -)] -#[derive(Serialize, Deserialize)] -#[serde(rename_all="kebab-case")] -/// Prune result. -pub struct PruneListItem { - pub backup_type: String, // enum - pub backup_id: String, - pub backup_time: i64, - /// Keep snapshot - pub keep: bool, -} - pub const PRUNE_SCHEMA_KEEP_DAILY: Schema = IntegerSchema::new( "Number of daily backups to keep.") .minimum(1) @@ -595,30 +387,6 @@ pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema = IntegerSchema::new( .minimum(1) .schema(); -#[api( - properties: { - "filename": { - schema: BACKUP_ARCHIVE_NAME_SCHEMA, - }, - "crypt-mode": { - type: CryptMode, - optional: true, - }, - }, -)] -#[derive(Serialize, Deserialize)] -#[serde(rename_all="kebab-case")] -/// Basic information about archive files inside a backup snapshot. -pub struct BackupContent { - pub filename: String, - /// Info if file is encrypted, signed, or neither. - #[serde(skip_serializing_if="Option::is_none")] - pub crypt_mode: Option, - /// Archive size (from backup manifest). - #[serde(skip_serializing_if="Option::is_none")] - pub size: Option, -} - #[api()] #[derive(Default, Serialize, Deserialize)] /// Storage space usage information. diff --git a/src/backup/mod.rs b/src/backup/mod.rs index b5c8867b..4161b402 100644 --- a/src/backup/mod.rs +++ b/src/backup/mod.rs @@ -77,6 +77,8 @@ pub use pbs_datastore::manifest::*; pub use pbs_datastore::prune; pub use pbs_datastore::prune::*; +pub use pbs_datastore::store_progress::StoreProgress; + pub use pbs_datastore::dynamic_index::*; pub use pbs_datastore::fixed_index; pub use pbs_datastore::fixed_index::*; @@ -97,9 +99,6 @@ pub use dynamic_index::*; mod datastore; pub use datastore::*; -mod store_progress; -pub use store_progress::*; - mod verify; pub use verify::*; diff --git a/src/client/pull.rs b/src/client/pull.rs index 42b94bc4..9b454341 100644 --- a/src/client/pull.rs +++ b/src/client/pull.rs @@ -12,13 +12,20 @@ use serde_json::json; use proxmox::api::error::{HttpError, StatusCode}; -use pbs_datastore::task_log; +use pbs_api_types::{Authid, SnapshotListItem, GroupListItem}; +use pbs_datastore::{task_log, BackupInfo, BackupDir, BackupGroup, StoreProgress}; +use pbs_datastore::data_blob::DataBlob; +use pbs_datastore::dynamic_index::DynamicIndexReader; +use pbs_datastore::fixed_index::FixedIndexReader; +use pbs_datastore::index::IndexFile; +use pbs_datastore::manifest::{ + CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, FileInfo, archive_type +}; use pbs_tools::sha::sha256; use crate::{ - api2::types::*, - backup::*, - client::*, + backup::DataStore, + client::{BackupReader, BackupRepository, HttpClient, HttpClientOptions, RemoteChunkReader}, server::WorkerTask, tools::ParallelHandler, };