2021-02-16 17:06:57 +00:00
|
|
|
use std::collections::HashSet;
|
2022-02-22 10:03:13 +00:00
|
|
|
use std::io::{self, Read, Seek, SeekFrom, Write};
|
2020-03-23 14:03:18 +00:00
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use std::pin::Pin;
|
|
|
|
use std::sync::{Arc, Mutex};
|
2020-06-24 09:57:12 +00:00
|
|
|
use std::task::Context;
|
2020-03-23 14:03:18 +00:00
|
|
|
|
|
|
|
use anyhow::{bail, format_err, Error};
|
|
|
|
use futures::stream::{StreamExt, TryStreamExt};
|
|
|
|
use serde_json::{json, Value};
|
|
|
|
use tokio::sync::mpsc;
|
2021-01-11 08:50:04 +00:00
|
|
|
use tokio_stream::wrappers::ReceiverStream;
|
2020-03-23 14:03:18 +00:00
|
|
|
use xdg::BaseDirectories;
|
2019-08-09 07:46:49 +00:00
|
|
|
|
2020-03-23 14:03:18 +00:00
|
|
|
use pathpatterns::{MatchEntry, MatchType, PatternFlag};
|
2022-02-22 10:03:13 +00:00
|
|
|
use proxmox_async::blocking::TokioWriterAdapter;
|
2021-11-25 11:30:03 +00:00
|
|
|
use proxmox_io::StdChannelWriter;
|
2022-02-22 10:03:13 +00:00
|
|
|
use proxmox_router::{cli::*, ApiMethod, RpcEnvironment};
|
2021-10-08 09:19:37 +00:00
|
|
|
use proxmox_schema::api;
|
2022-02-22 10:03:13 +00:00
|
|
|
use proxmox_sys::fs::{file_get_json, image_size, replace_file, CreateOptions};
|
|
|
|
use proxmox_time::{epoch_i64, strftime_local};
|
2020-06-24 09:57:12 +00:00
|
|
|
use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
|
2018-12-14 07:28:56 +00:00
|
|
|
|
2021-07-19 12:59:51 +00:00
|
|
|
use pbs_api_types::{
|
2022-02-22 10:03:13 +00:00
|
|
|
Authid, CryptMode, Fingerprint, GroupListItem, HumanByte, PruneListItem, PruneOptions,
|
|
|
|
RateLimitConfig, SnapshotListItem, StorageStatus, BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA,
|
|
|
|
BACKUP_TYPE_SCHEMA, TRAFFIC_CONTROL_BURST_SCHEMA, TRAFFIC_CONTROL_RATE_SCHEMA,
|
2021-07-19 08:50:18 +00:00
|
|
|
};
|
|
|
|
use pbs_client::catalog_shell::Shell;
|
|
|
|
use pbs_client::tools::{
|
|
|
|
complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot,
|
|
|
|
complete_backup_source, complete_chunk_size, complete_group_or_snapshot,
|
|
|
|
complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect,
|
2021-11-03 12:56:06 +00:00
|
|
|
connect_rate_limited, extract_repository_from_value,
|
2021-07-19 08:50:18 +00:00
|
|
|
key_source::{
|
|
|
|
crypto_parameters, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
|
|
|
|
KEYFILE_SCHEMA, MASTER_PUBKEY_FD_SCHEMA, MASTER_PUBKEY_FILE_SCHEMA,
|
|
|
|
},
|
|
|
|
CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA,
|
|
|
|
};
|
2022-02-22 10:03:13 +00:00
|
|
|
use pbs_client::{
|
|
|
|
delete_ticket_info, parse_backup_specification, view_task_result, BackupReader,
|
|
|
|
BackupRepository, BackupSpecificationType, BackupStats, BackupWriter, ChunkStream,
|
|
|
|
FixedChunkStream, HttpClient, PxarBackupStream, RemoteChunkReader, UploadOptions,
|
|
|
|
BACKUP_SOURCE_SCHEMA,
|
|
|
|
};
|
|
|
|
use pbs_config::key_config::{decrypt_key, rsa_encrypt_key_config, KeyConfig};
|
2021-07-19 08:50:18 +00:00
|
|
|
use pbs_datastore::backup_info::{BackupDir, BackupGroup};
|
2021-07-19 12:59:51 +00:00
|
|
|
use pbs_datastore::catalog::{BackupCatalogWriter, CatalogReader, CatalogWriter};
|
|
|
|
use pbs_datastore::chunk_store::verify_chunk_size;
|
2021-07-21 12:12:22 +00:00
|
|
|
use pbs_datastore::dynamic_index::{BufferedDynamicReader, DynamicIndexReader};
|
2021-07-19 08:50:18 +00:00
|
|
|
use pbs_datastore::fixed_index::FixedIndexReader;
|
|
|
|
use pbs_datastore::index::IndexFile;
|
2021-07-19 12:59:51 +00:00
|
|
|
use pbs_datastore::manifest::{
|
2022-02-22 10:03:13 +00:00
|
|
|
archive_type, ArchiveType, BackupManifest, ENCRYPTED_KEY_BLOB_NAME, MANIFEST_BLOB_NAME,
|
2021-07-19 12:59:51 +00:00
|
|
|
};
|
2021-07-19 08:50:18 +00:00
|
|
|
use pbs_datastore::read_chunk::AsyncReadChunk;
|
2022-02-22 10:03:13 +00:00
|
|
|
use pbs_datastore::CATALOG_NAME;
|
2021-09-07 07:22:14 +00:00
|
|
|
use pbs_tools::crypt_config::CryptConfig;
|
2022-02-22 10:03:13 +00:00
|
|
|
use pbs_tools::json;
|
2021-07-06 10:49:10 +00:00
|
|
|
|
2021-08-26 09:00:37 +00:00
|
|
|
mod benchmark;
|
|
|
|
pub use benchmark::*;
|
|
|
|
mod mount;
|
|
|
|
pub use mount::*;
|
|
|
|
mod task;
|
|
|
|
pub use task::*;
|
|
|
|
mod catalog;
|
|
|
|
pub use catalog::*;
|
|
|
|
mod snapshot;
|
|
|
|
pub use snapshot::*;
|
|
|
|
pub mod key;
|
2020-07-02 12:00:32 +00:00
|
|
|
|
2019-03-13 08:47:12 +00:00
|
|
|
fn record_repository(repo: &BackupRepository) {
|
|
|
|
let base = match BaseDirectories::with_prefix("proxmox-backup") {
|
|
|
|
Ok(v) => v,
|
|
|
|
_ => return,
|
|
|
|
};
|
|
|
|
|
|
|
|
// usually $HOME/.cache/proxmox-backup/repo-list
|
|
|
|
let path = match base.place_cache_file("repo-list") {
|
|
|
|
Ok(v) => v,
|
|
|
|
_ => return,
|
|
|
|
};
|
|
|
|
|
2019-10-25 16:04:37 +00:00
|
|
|
let mut data = file_get_json(&path, None).unwrap_or_else(|_| json!({}));
|
2019-03-13 08:47:12 +00:00
|
|
|
|
|
|
|
let repo = repo.to_string();
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
data[&repo] = json! { data[&repo].as_i64().unwrap_or(0) + 1 };
|
2019-03-13 08:47:12 +00:00
|
|
|
|
|
|
|
let mut map = serde_json::map::Map::new();
|
|
|
|
|
|
|
|
loop {
|
|
|
|
let mut max_used = 0;
|
|
|
|
let mut max_repo = None;
|
|
|
|
for (repo, count) in data.as_object().unwrap() {
|
2022-02-22 10:03:13 +00:00
|
|
|
if map.contains_key(repo) {
|
|
|
|
continue;
|
|
|
|
}
|
2019-03-13 08:47:12 +00:00
|
|
|
if let Some(count) = count.as_i64() {
|
|
|
|
if count > max_used {
|
|
|
|
max_used = count;
|
|
|
|
max_repo = Some(repo);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if let Some(repo) = max_repo {
|
|
|
|
map.insert(repo.to_owned(), json!(max_used));
|
|
|
|
} else {
|
|
|
|
break;
|
|
|
|
}
|
2022-02-22 10:03:13 +00:00
|
|
|
if map.len() > 10 {
|
|
|
|
// store max. 10 repos
|
2019-03-13 08:47:12 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
let new_data = json!(map);
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let _ = replace_file(
|
|
|
|
path,
|
|
|
|
new_data.to_string().as_bytes(),
|
|
|
|
CreateOptions::new(),
|
|
|
|
false,
|
|
|
|
);
|
2019-03-13 08:47:12 +00:00
|
|
|
}
|
|
|
|
|
2020-01-07 14:07:57 +00:00
|
|
|
async fn api_datastore_list_snapshots(
|
|
|
|
client: &HttpClient,
|
|
|
|
store: &str,
|
|
|
|
group: Option<BackupGroup>,
|
2020-02-27 10:27:44 +00:00
|
|
|
) -> Result<Value, Error> {
|
2020-01-07 14:07:57 +00:00
|
|
|
let path = format!("api2/json/admin/datastore/{}/snapshots", store);
|
|
|
|
|
|
|
|
let mut args = json!({});
|
|
|
|
if let Some(group) = group {
|
|
|
|
args["backup-type"] = group.backup_type().into();
|
|
|
|
args["backup-id"] = group.backup_id().into();
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut result = client.get(&path, Some(args)).await?;
|
|
|
|
|
2020-02-27 10:27:44 +00:00
|
|
|
Ok(result["data"].take())
|
2020-01-07 14:07:57 +00:00
|
|
|
}
|
|
|
|
|
2020-07-02 15:49:08 +00:00
|
|
|
pub async fn api_datastore_latest_snapshot(
|
2020-01-07 14:18:36 +00:00
|
|
|
client: &HttpClient,
|
|
|
|
store: &str,
|
|
|
|
group: BackupGroup,
|
2020-09-12 13:10:47 +00:00
|
|
|
) -> Result<(String, String, i64), Error> {
|
2020-02-27 10:27:44 +00:00
|
|
|
let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
|
|
|
|
let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
|
2020-01-07 14:18:36 +00:00
|
|
|
|
|
|
|
if list.is_empty() {
|
2022-02-22 10:03:13 +00:00
|
|
|
bail!(
|
|
|
|
"backup group {:?} does not contain any snapshots.",
|
|
|
|
group.group_path()
|
|
|
|
);
|
2020-01-07 14:18:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
|
|
|
|
|
2020-09-12 13:10:47 +00:00
|
|
|
let backup_time = list[0].backup_time;
|
2020-01-07 14:18:36 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
Ok((
|
|
|
|
group.backup_type().to_owned(),
|
|
|
|
group.backup_id().to_owned(),
|
|
|
|
backup_time,
|
|
|
|
))
|
2020-01-07 14:18:36 +00:00
|
|
|
}
|
|
|
|
|
2019-08-28 15:20:32 +00:00
|
|
|
async fn backup_directory<P: AsRef<Path>>(
|
2019-10-12 11:53:11 +00:00
|
|
|
client: &BackupWriter,
|
2019-03-01 08:35:41 +00:00
|
|
|
dir_path: P,
|
2019-02-19 14:19:12 +00:00
|
|
|
archive_name: &str,
|
2019-05-30 11:28:24 +00:00
|
|
|
chunk_size: Option<usize>,
|
2021-11-25 10:27:20 +00:00
|
|
|
catalog: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter<Error>>>>>,
|
2021-07-19 08:50:18 +00:00
|
|
|
pxar_create_options: pbs_client::pxar::PxarCreateOptions,
|
2021-01-25 13:42:52 +00:00
|
|
|
upload_options: UploadOptions,
|
2019-08-01 10:39:02 +00:00
|
|
|
) -> Result<BackupStats, Error> {
|
2022-02-22 10:03:13 +00:00
|
|
|
let pxar_stream = PxarBackupStream::open(dir_path.as_ref(), catalog, pxar_create_options)?;
|
2019-08-28 15:20:32 +00:00
|
|
|
let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
|
2019-01-18 11:01:37 +00:00
|
|
|
|
2020-12-04 10:53:34 +00:00
|
|
|
let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
|
2019-01-02 10:02:56 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let stream = ReceiverStream::new(rx).map_err(Error::from);
|
2019-03-01 08:35:41 +00:00
|
|
|
|
2019-05-28 08:12:44 +00:00
|
|
|
// spawn chunker inside a separate task so that it can run parallel
|
2019-08-28 15:20:32 +00:00
|
|
|
tokio::spawn(async move {
|
2019-12-12 14:27:07 +00:00
|
|
|
while let Some(v) = chunk_stream.next().await {
|
|
|
|
let _ = tx.send(v).await;
|
|
|
|
}
|
2019-08-28 15:20:32 +00:00
|
|
|
});
|
2019-03-01 08:35:41 +00:00
|
|
|
|
2021-01-25 13:42:52 +00:00
|
|
|
if upload_options.fixed_size.is_some() {
|
|
|
|
bail!("cannot backup directory with fixed chunk size!");
|
|
|
|
}
|
|
|
|
|
2019-08-28 15:20:32 +00:00
|
|
|
let stats = client
|
2021-01-25 13:42:52 +00:00
|
|
|
.upload_stream(archive_name, stream, upload_options)
|
2019-08-28 15:20:32 +00:00
|
|
|
.await?;
|
2018-12-27 09:11:11 +00:00
|
|
|
|
2019-08-01 10:39:02 +00:00
|
|
|
Ok(stats)
|
2018-12-27 09:11:11 +00:00
|
|
|
}
|
|
|
|
|
2019-08-28 15:20:32 +00:00
|
|
|
async fn backup_image<P: AsRef<Path>>(
|
2019-10-12 11:53:11 +00:00
|
|
|
client: &BackupWriter,
|
2019-05-30 10:46:01 +00:00
|
|
|
image_path: P,
|
|
|
|
archive_name: &str,
|
2019-05-30 11:28:24 +00:00
|
|
|
chunk_size: Option<usize>,
|
2021-01-25 13:42:52 +00:00
|
|
|
upload_options: UploadOptions,
|
2019-08-01 10:39:02 +00:00
|
|
|
) -> Result<BackupStats, Error> {
|
2019-05-30 10:46:01 +00:00
|
|
|
let path = image_path.as_ref().to_owned();
|
|
|
|
|
2019-08-28 15:20:32 +00:00
|
|
|
let file = tokio::fs::File::open(path).await?;
|
2019-05-30 10:46:01 +00:00
|
|
|
|
2019-12-12 14:27:07 +00:00
|
|
|
let stream = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
|
2019-05-30 10:46:01 +00:00
|
|
|
.map_err(Error::from);
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4 * 1024 * 1024));
|
2019-05-30 10:46:01 +00:00
|
|
|
|
2021-01-25 13:42:52 +00:00
|
|
|
if upload_options.fixed_size.is_none() {
|
|
|
|
bail!("cannot backup image with dynamic chunk size!");
|
|
|
|
}
|
|
|
|
|
2019-08-28 15:20:32 +00:00
|
|
|
let stats = client
|
2021-01-25 13:42:52 +00:00
|
|
|
.upload_stream(archive_name, stream, upload_options)
|
2019-08-28 15:20:32 +00:00
|
|
|
.await?;
|
2019-05-30 10:46:01 +00:00
|
|
|
|
2019-08-01 10:39:02 +00:00
|
|
|
Ok(stats)
|
2019-05-30 10:46:01 +00:00
|
|
|
}
|
|
|
|
|
2019-12-16 12:34:49 +00:00
|
|
|
#[api(
|
|
|
|
input: {
|
|
|
|
properties: {
|
|
|
|
repository: {
|
|
|
|
schema: REPO_URL_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
"output-format": {
|
|
|
|
schema: OUTPUT_FORMAT,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
)]
|
|
|
|
/// List backup groups.
|
|
|
|
async fn list_backup_groups(param: Value) -> Result<Value, Error> {
|
2020-02-27 12:31:23 +00:00
|
|
|
let output_format = get_output_format(¶m);
|
|
|
|
|
2019-07-16 05:30:04 +00:00
|
|
|
let repo = extract_repository_from_value(¶m)?;
|
2019-03-02 10:29:05 +00:00
|
|
|
|
2020-11-10 10:54:50 +00:00
|
|
|
let client = connect(&repo)?;
|
2019-03-02 10:29:05 +00:00
|
|
|
|
2019-03-13 08:47:12 +00:00
|
|
|
let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
|
2019-03-02 10:29:05 +00:00
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
let mut result = client.get(&path, None).await?;
|
2019-03-02 10:29:05 +00:00
|
|
|
|
2019-03-13 08:47:12 +00:00
|
|
|
record_repository(&repo);
|
|
|
|
|
2020-02-27 12:31:23 +00:00
|
|
|
let render_group_path = |_v: &Value, record: &Value| -> Result<String, Error> {
|
|
|
|
let item: GroupListItem = serde_json::from_value(record.to_owned())?;
|
|
|
|
let group = BackupGroup::new(item.backup_type, item.backup_id);
|
|
|
|
Ok(group.group_path().to_str().unwrap().to_owned())
|
|
|
|
};
|
2019-03-02 10:29:05 +00:00
|
|
|
|
2020-02-28 08:09:22 +00:00
|
|
|
let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
|
|
|
|
let item: GroupListItem = serde_json::from_value(record.to_owned())?;
|
2020-09-11 12:34:38 +00:00
|
|
|
let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup)?;
|
2020-02-28 08:09:22 +00:00
|
|
|
Ok(snapshot.relative_path().to_str().unwrap().to_owned())
|
2020-02-27 12:31:23 +00:00
|
|
|
};
|
2019-03-02 10:29:05 +00:00
|
|
|
|
2020-02-27 12:31:23 +00:00
|
|
|
let render_files = |_v: &Value, record: &Value| -> Result<String, Error> {
|
|
|
|
let item: GroupListItem = serde_json::from_value(record.to_owned())?;
|
2021-07-06 11:26:35 +00:00
|
|
|
Ok(pbs_tools::format::render_backup_file_list(&item.files))
|
2020-02-27 12:31:23 +00:00
|
|
|
};
|
2019-03-02 10:29:05 +00:00
|
|
|
|
2020-02-27 12:31:23 +00:00
|
|
|
let options = default_table_format_options()
|
|
|
|
.sortby("backup-type", false)
|
|
|
|
.sortby("backup-id", false)
|
2022-02-22 10:03:13 +00:00
|
|
|
.column(
|
|
|
|
ColumnConfig::new("backup-id")
|
|
|
|
.renderer(render_group_path)
|
|
|
|
.header("group"),
|
|
|
|
)
|
2020-02-28 08:09:22 +00:00
|
|
|
.column(
|
|
|
|
ColumnConfig::new("last-backup")
|
|
|
|
.renderer(render_last_backup)
|
|
|
|
.header("last snapshot")
|
2022-02-22 10:03:13 +00:00
|
|
|
.right_align(false),
|
2020-02-28 08:09:22 +00:00
|
|
|
)
|
2020-02-27 12:31:23 +00:00
|
|
|
.column(ColumnConfig::new("backup-count"))
|
|
|
|
.column(ColumnConfig::new("files").renderer(render_files));
|
2019-03-02 15:28:36 +00:00
|
|
|
|
2020-02-27 12:31:23 +00:00
|
|
|
let mut data: Value = result["data"].take();
|
2019-03-02 15:28:36 +00:00
|
|
|
|
2021-08-26 09:00:37 +00:00
|
|
|
let return_type = &pbs_api_types::ADMIN_DATASTORE_LIST_GROUPS_RETURN_TYPE;
|
2019-03-02 10:29:05 +00:00
|
|
|
|
2020-12-18 11:26:07 +00:00
|
|
|
format_and_print_result_full(&mut data, return_type, &output_format, &options);
|
2019-07-16 11:35:25 +00:00
|
|
|
|
2019-03-02 10:29:05 +00:00
|
|
|
Ok(Value::Null)
|
|
|
|
}
|
|
|
|
|
2020-10-13 08:58:41 +00:00
|
|
|
#[api(
|
|
|
|
input: {
|
|
|
|
properties: {
|
|
|
|
repository: {
|
|
|
|
schema: REPO_URL_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
group: {
|
|
|
|
type: String,
|
|
|
|
description: "Backup group.",
|
|
|
|
},
|
|
|
|
"new-owner": {
|
2020-10-23 11:33:21 +00:00
|
|
|
type: Authid,
|
2020-10-13 08:58:41 +00:00
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
)]
|
|
|
|
/// Change owner of a backup group
|
|
|
|
async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Error> {
|
|
|
|
let repo = extract_repository_from_value(¶m)?;
|
|
|
|
|
2021-12-04 13:11:55 +00:00
|
|
|
let client = connect(&repo)?;
|
2020-10-13 08:58:41 +00:00
|
|
|
|
|
|
|
param.as_object_mut().unwrap().remove("repository");
|
|
|
|
|
|
|
|
let group: BackupGroup = group.parse()?;
|
|
|
|
|
|
|
|
param["backup-type"] = group.backup_type().into();
|
|
|
|
param["backup-id"] = group.backup_id().into();
|
|
|
|
|
|
|
|
let path = format!("api2/json/admin/datastore/{}/change-owner", repo.store());
|
|
|
|
client.post(&path, Some(param)).await?;
|
|
|
|
|
|
|
|
record_repository(&repo);
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-12-16 12:34:49 +00:00
|
|
|
#[api(
|
|
|
|
input: {
|
|
|
|
properties: {
|
|
|
|
repository: {
|
|
|
|
schema: REPO_URL_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
)]
|
|
|
|
/// Try to login. If successful, store ticket.
|
|
|
|
async fn api_login(param: Value) -> Result<Value, Error> {
|
2019-08-10 07:12:17 +00:00
|
|
|
let repo = extract_repository_from_value(¶m)?;
|
|
|
|
|
2020-11-10 10:54:50 +00:00
|
|
|
let client = connect(&repo)?;
|
2019-12-16 09:06:26 +00:00
|
|
|
client.login().await?;
|
2019-08-10 07:12:17 +00:00
|
|
|
|
|
|
|
record_repository(&repo);
|
|
|
|
|
|
|
|
Ok(Value::Null)
|
|
|
|
}
|
|
|
|
|
2019-12-16 12:34:49 +00:00
|
|
|
#[api(
|
|
|
|
input: {
|
|
|
|
properties: {
|
|
|
|
repository: {
|
|
|
|
schema: REPO_URL_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
)]
|
|
|
|
/// Logout (delete stored ticket).
|
|
|
|
fn api_logout(param: Value) -> Result<Value, Error> {
|
2019-08-10 07:12:17 +00:00
|
|
|
let repo = extract_repository_from_value(¶m)?;
|
|
|
|
|
2020-01-27 08:34:02 +00:00
|
|
|
delete_ticket_info("proxmox-backup", repo.host(), repo.user())?;
|
2019-08-10 07:12:17 +00:00
|
|
|
|
|
|
|
Ok(Value::Null)
|
|
|
|
}
|
|
|
|
|
2020-07-10 07:34:07 +00:00
|
|
|
#[api(
|
|
|
|
input: {
|
|
|
|
properties: {
|
|
|
|
repository: {
|
|
|
|
schema: REPO_URL_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
"output-format": {
|
|
|
|
schema: OUTPUT_FORMAT,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
)]
|
|
|
|
/// Show client and optional server version
|
|
|
|
async fn api_version(param: Value) -> Result<(), Error> {
|
|
|
|
let output_format = get_output_format(¶m);
|
|
|
|
|
|
|
|
let mut version_info = json!({
|
|
|
|
"client": {
|
2021-07-19 12:59:17 +00:00
|
|
|
"version": pbs_buildcfg::PROXMOX_PKG_VERSION,
|
|
|
|
"release": pbs_buildcfg::PROXMOX_PKG_RELEASE,
|
|
|
|
"repoid": pbs_buildcfg::PROXMOX_PKG_REPOID,
|
2020-07-10 07:34:07 +00:00
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
let repo = extract_repository_from_value(¶m);
|
|
|
|
if let Ok(repo) = repo {
|
2020-11-10 10:54:50 +00:00
|
|
|
let client = connect(&repo)?;
|
2020-07-10 07:34:07 +00:00
|
|
|
|
|
|
|
match client.get("api2/json/version", None).await {
|
|
|
|
Ok(mut result) => version_info["server"] = result["data"].take(),
|
|
|
|
Err(e) => eprintln!("could not connect to server - {}", e),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if output_format == "text" {
|
2021-07-19 12:59:17 +00:00
|
|
|
println!(
|
|
|
|
"client version: {}.{}",
|
|
|
|
pbs_buildcfg::PROXMOX_PKG_VERSION,
|
|
|
|
pbs_buildcfg::PROXMOX_PKG_RELEASE,
|
|
|
|
);
|
2020-07-10 07:34:07 +00:00
|
|
|
if let Some(server) = version_info["server"].as_object() {
|
|
|
|
let server_version = server["version"].as_str().unwrap();
|
|
|
|
let server_release = server["release"].as_str().unwrap();
|
|
|
|
println!("server version: {}.{}", server_version, server_release);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
format_and_print_result(&version_info, &output_format);
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-12-16 12:34:49 +00:00
|
|
|
#[api(
|
2019-12-19 06:57:53 +00:00
|
|
|
input: {
|
2019-12-16 12:34:49 +00:00
|
|
|
properties: {
|
|
|
|
repository: {
|
|
|
|
schema: REPO_URL_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
2019-12-19 06:57:53 +00:00
|
|
|
"output-format": {
|
|
|
|
schema: OUTPUT_FORMAT,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
2019-12-16 12:34:49 +00:00
|
|
|
)]
|
|
|
|
/// Start garbage collection for a specific repository.
|
|
|
|
async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
|
2019-07-16 05:30:04 +00:00
|
|
|
let repo = extract_repository_from_value(¶m)?;
|
2020-02-27 11:41:15 +00:00
|
|
|
|
|
|
|
let output_format = get_output_format(¶m);
|
2019-02-20 13:10:45 +00:00
|
|
|
|
2021-12-04 13:11:55 +00:00
|
|
|
let client = connect(&repo)?;
|
2019-02-20 13:10:45 +00:00
|
|
|
|
2019-03-13 08:47:12 +00:00
|
|
|
let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
|
2019-02-20 13:10:45 +00:00
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
let result = client.post(&path, None).await?;
|
2019-02-20 13:10:45 +00:00
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
record_repository(&repo);
|
2019-03-13 08:47:12 +00:00
|
|
|
|
2021-12-04 13:11:55 +00:00
|
|
|
view_task_result(&client, result, &output_format).await?;
|
2019-12-09 12:29:23 +00:00
|
|
|
|
|
|
|
Ok(Value::Null)
|
2019-02-20 13:10:45 +00:00
|
|
|
}
|
2019-02-13 11:30:52 +00:00
|
|
|
|
2021-01-25 13:42:48 +00:00
|
|
|
struct CatalogUploadResult {
|
2021-11-25 10:27:20 +00:00
|
|
|
catalog_writer: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter<Error>>>>>,
|
2021-01-25 13:42:48 +00:00
|
|
|
result: tokio::sync::oneshot::Receiver<Result<BackupStats, Error>>,
|
|
|
|
}
|
|
|
|
|
2019-11-08 09:35:48 +00:00
|
|
|
fn spawn_catalog_upload(
|
2020-07-08 08:42:20 +00:00
|
|
|
client: Arc<BackupWriter>,
|
2020-07-08 12:06:50 +00:00
|
|
|
encrypt: bool,
|
2021-01-25 13:42:48 +00:00
|
|
|
) -> Result<CatalogUploadResult, Error> {
|
2020-01-22 11:49:08 +00:00
|
|
|
let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes
|
2021-11-19 16:36:06 +00:00
|
|
|
let catalog_stream = proxmox_async::blocking::StdChannelStream(catalog_rx);
|
2022-02-22 10:03:13 +00:00
|
|
|
let catalog_chunk_size = 512 * 1024;
|
2019-11-08 09:35:48 +00:00
|
|
|
let catalog_chunk_stream = ChunkStream::new(catalog_stream, Some(catalog_chunk_size));
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let catalog_writer = Arc::new(Mutex::new(CatalogWriter::new(TokioWriterAdapter::new(
|
|
|
|
StdChannelWriter::new(catalog_tx),
|
|
|
|
))?));
|
2019-11-08 09:35:48 +00:00
|
|
|
|
|
|
|
let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel();
|
|
|
|
|
2021-01-25 13:42:52 +00:00
|
|
|
let upload_options = UploadOptions {
|
|
|
|
encrypt,
|
|
|
|
compress: true,
|
|
|
|
..UploadOptions::default()
|
|
|
|
};
|
|
|
|
|
2019-11-08 09:35:48 +00:00
|
|
|
tokio::spawn(async move {
|
|
|
|
let catalog_upload_result = client
|
2021-01-25 13:42:52 +00:00
|
|
|
.upload_stream(CATALOG_NAME, catalog_chunk_stream, upload_options)
|
2019-11-08 09:35:48 +00:00
|
|
|
.await;
|
|
|
|
|
|
|
|
if let Err(ref err) = catalog_upload_result {
|
|
|
|
eprintln!("catalog upload error - {}", err);
|
|
|
|
client.cancel();
|
|
|
|
}
|
|
|
|
|
|
|
|
let _ = catalog_result_tx.send(catalog_upload_result);
|
|
|
|
});
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
Ok(CatalogUploadResult {
|
|
|
|
catalog_writer,
|
|
|
|
result: catalog_result_rx,
|
|
|
|
})
|
2019-11-08 09:35:48 +00:00
|
|
|
}
|
|
|
|
|
2019-12-16 12:34:49 +00:00
|
|
|
#[api(
|
|
|
|
input: {
|
|
|
|
properties: {
|
|
|
|
backupspec: {
|
|
|
|
type: Array,
|
|
|
|
description: "List of backup source specifications ([<label.ext>:<path>] ...)",
|
|
|
|
items: {
|
|
|
|
schema: BACKUP_SOURCE_SCHEMA,
|
|
|
|
}
|
|
|
|
},
|
|
|
|
repository: {
|
|
|
|
schema: REPO_URL_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
"include-dev": {
|
|
|
|
description: "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.",
|
|
|
|
optional: true,
|
|
|
|
items: {
|
|
|
|
type: String,
|
|
|
|
description: "Path to file.",
|
|
|
|
}
|
|
|
|
},
|
2020-11-16 09:22:47 +00:00
|
|
|
"all-file-systems": {
|
|
|
|
type: Boolean,
|
|
|
|
description: "Include all mounted subdirectories.",
|
|
|
|
optional: true,
|
2022-02-18 14:01:09 +00:00
|
|
|
default: false,
|
2020-11-16 09:22:47 +00:00
|
|
|
},
|
2019-12-16 12:34:49 +00:00
|
|
|
keyfile: {
|
|
|
|
schema: KEYFILE_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
2020-07-08 11:52:17 +00:00
|
|
|
"keyfd": {
|
|
|
|
schema: KEYFD_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
2021-02-05 15:35:32 +00:00
|
|
|
"master-pubkey-file": {
|
|
|
|
schema: MASTER_PUBKEY_FILE_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
"master-pubkey-fd": {
|
|
|
|
schema: MASTER_PUBKEY_FD_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
2020-07-08 08:09:15 +00:00
|
|
|
"crypt-mode": {
|
|
|
|
type: CryptMode,
|
2020-07-06 12:35:28 +00:00
|
|
|
optional: true,
|
|
|
|
},
|
2019-12-16 12:34:49 +00:00
|
|
|
"skip-lost-and-found": {
|
|
|
|
type: Boolean,
|
|
|
|
description: "Skip lost+found directory.",
|
|
|
|
optional: true,
|
2022-02-18 14:01:09 +00:00
|
|
|
default: false,
|
2019-12-16 12:34:49 +00:00
|
|
|
},
|
|
|
|
"backup-type": {
|
|
|
|
schema: BACKUP_TYPE_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
"backup-id": {
|
|
|
|
schema: BACKUP_ID_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
"backup-time": {
|
|
|
|
schema: BACKUP_TIME_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
"chunk-size": {
|
|
|
|
schema: CHUNK_SIZE_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
2021-11-03 12:56:06 +00:00
|
|
|
rate: {
|
2021-11-06 17:46:58 +00:00
|
|
|
schema: TRAFFIC_CONTROL_RATE_SCHEMA,
|
2021-11-03 12:56:06 +00:00
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
burst: {
|
2021-11-06 17:46:58 +00:00
|
|
|
schema: TRAFFIC_CONTROL_BURST_SCHEMA,
|
2021-11-03 12:56:06 +00:00
|
|
|
optional: true,
|
|
|
|
},
|
2020-02-27 12:13:12 +00:00
|
|
|
"exclude": {
|
|
|
|
type: Array,
|
|
|
|
description: "List of paths or patterns for matching files to exclude.",
|
|
|
|
optional: true,
|
|
|
|
items: {
|
|
|
|
type: String,
|
|
|
|
description: "Path or match pattern.",
|
|
|
|
}
|
|
|
|
},
|
2020-01-10 11:50:06 +00:00
|
|
|
"entries-max": {
|
|
|
|
type: Integer,
|
|
|
|
description: "Max number of entries to hold in memory.",
|
|
|
|
optional: true,
|
2021-07-19 08:50:18 +00:00
|
|
|
default: pbs_client::pxar::ENCODER_MAX_ENTRIES as isize,
|
2020-01-10 11:50:06 +00:00
|
|
|
},
|
2020-02-24 11:48:40 +00:00
|
|
|
"verbose": {
|
|
|
|
type: Boolean,
|
|
|
|
description: "Verbose output.",
|
|
|
|
optional: true,
|
2022-02-18 14:01:09 +00:00
|
|
|
default: false,
|
2020-02-24 11:48:40 +00:00
|
|
|
},
|
2022-02-18 09:55:38 +00:00
|
|
|
"dry-run": {
|
|
|
|
type: Boolean,
|
|
|
|
description: "Just show what backup would do, but do not upload anything.",
|
|
|
|
optional: true,
|
2022-02-18 14:01:09 +00:00
|
|
|
default: false,
|
2022-02-18 09:55:38 +00:00
|
|
|
},
|
2019-12-16 12:34:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
)]
|
|
|
|
/// Create (host) backup.
|
|
|
|
async fn create_backup(
|
2019-01-26 13:50:37 +00:00
|
|
|
param: Value,
|
2022-02-18 14:01:09 +00:00
|
|
|
all_file_systems: bool,
|
|
|
|
skip_lost_and_found: bool,
|
|
|
|
dry_run: bool,
|
|
|
|
verbose: bool,
|
2019-01-26 13:50:37 +00:00
|
|
|
_info: &ApiMethod,
|
2019-06-07 11:10:56 +00:00
|
|
|
_rpcenv: &mut dyn RpcEnvironment,
|
2019-01-26 13:50:37 +00:00
|
|
|
) -> Result<Value, Error> {
|
2019-07-16 05:30:04 +00:00
|
|
|
let repo = extract_repository_from_value(¶m)?;
|
2019-02-26 11:27:28 +00:00
|
|
|
|
2021-07-20 09:06:53 +00:00
|
|
|
let backupspec_list = json::required_array_param(¶m, "backupspec")?;
|
2018-12-14 12:39:41 +00:00
|
|
|
|
2019-07-25 11:44:01 +00:00
|
|
|
let backup_time_opt = param["backup-time"].as_i64();
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let chunk_size_opt = param["chunk-size"].as_u64().map(|v| (v * 1024) as usize);
|
2018-12-21 10:18:08 +00:00
|
|
|
|
2019-02-19 14:19:12 +00:00
|
|
|
if let Some(size) = chunk_size_opt {
|
|
|
|
verify_chunk_size(size)?;
|
2018-12-21 10:18:08 +00:00
|
|
|
}
|
|
|
|
|
2021-11-22 05:26:55 +00:00
|
|
|
let rate = match param["rate"].as_str() {
|
|
|
|
Some(s) => Some(s.parse::<HumanByte>()?),
|
|
|
|
None => None,
|
|
|
|
};
|
|
|
|
let burst = match param["burst"].as_str() {
|
|
|
|
Some(s) => Some(s.parse::<HumanByte>()?),
|
|
|
|
None => None,
|
|
|
|
};
|
|
|
|
|
|
|
|
let rate_limit = RateLimitConfig::with_same_inout(rate, burst);
|
2021-11-03 12:56:06 +00:00
|
|
|
|
2021-02-05 15:35:31 +00:00
|
|
|
let crypto = crypto_parameters(¶m)?;
|
2019-06-19 15:16:41 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let backup_id = param["backup-id"]
|
|
|
|
.as_str()
|
|
|
|
.unwrap_or(proxmox_sys::nodename());
|
2019-03-06 06:02:52 +00:00
|
|
|
|
2019-07-26 07:07:29 +00:00
|
|
|
let backup_type = param["backup-type"].as_str().unwrap_or("host");
|
2019-07-25 11:44:01 +00:00
|
|
|
|
2019-07-24 05:48:59 +00:00
|
|
|
let include_dev = param["include-dev"].as_array();
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let entries_max = param["entries-max"]
|
|
|
|
.as_u64()
|
2021-07-19 08:50:18 +00:00
|
|
|
.unwrap_or(pbs_client::pxar::ENCODER_MAX_ENTRIES as u64);
|
2020-01-10 11:50:06 +00:00
|
|
|
|
2020-02-27 12:13:12 +00:00
|
|
|
let empty = Vec::new();
|
2020-03-23 14:03:18 +00:00
|
|
|
let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
|
|
|
|
|
2020-06-09 11:17:55 +00:00
|
|
|
let mut pattern_list = Vec::with_capacity(exclude_args.len());
|
2020-03-23 14:03:18 +00:00
|
|
|
for entry in exclude_args {
|
2022-02-22 10:03:13 +00:00
|
|
|
let entry = entry
|
|
|
|
.as_str()
|
|
|
|
.ok_or_else(|| format_err!("Invalid pattern string slice"))?;
|
2020-06-09 11:17:55 +00:00
|
|
|
pattern_list.push(
|
2020-03-23 14:03:18 +00:00
|
|
|
MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
|
2022-02-22 10:03:13 +00:00
|
|
|
.map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?,
|
2020-03-23 14:03:18 +00:00
|
|
|
);
|
2020-02-27 12:13:12 +00:00
|
|
|
}
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let mut devices = if all_file_systems {
|
|
|
|
None
|
|
|
|
} else {
|
|
|
|
Some(HashSet::new())
|
|
|
|
};
|
2019-07-24 05:48:59 +00:00
|
|
|
|
|
|
|
if let Some(include_dev) = include_dev {
|
|
|
|
if all_file_systems {
|
|
|
|
bail!("option 'all-file-systems' conflicts with option 'include-dev'");
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut set = HashSet::new();
|
|
|
|
for path in include_dev {
|
|
|
|
let path = path.as_str().unwrap();
|
|
|
|
let stat = nix::sys::stat::stat(path)
|
|
|
|
.map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
|
|
|
|
set.insert(stat.st_dev);
|
|
|
|
}
|
|
|
|
devices = Some(set);
|
|
|
|
}
|
|
|
|
|
2019-02-26 11:27:28 +00:00
|
|
|
let mut upload_list = vec![];
|
2020-07-23 16:16:36 +00:00
|
|
|
let mut target_set = HashSet::new();
|
2018-12-14 12:39:41 +00:00
|
|
|
|
2019-02-26 11:27:28 +00:00
|
|
|
for backupspec in backupspec_list {
|
2020-05-30 08:54:38 +00:00
|
|
|
let spec = parse_backup_specification(backupspec.as_str().unwrap())?;
|
|
|
|
let filename = &spec.config_string;
|
|
|
|
let target = &spec.archive_name;
|
2018-12-27 09:11:11 +00:00
|
|
|
|
2020-07-23 16:16:36 +00:00
|
|
|
if target_set.contains(target) {
|
|
|
|
bail!("got target twice: '{}'", target);
|
|
|
|
}
|
|
|
|
target_set.insert(target.to_string());
|
|
|
|
|
2019-05-30 10:19:23 +00:00
|
|
|
use std::os::unix::fs::FileTypeExt;
|
|
|
|
|
2019-09-03 14:17:23 +00:00
|
|
|
let metadata = std::fs::metadata(filename)
|
|
|
|
.map_err(|err| format_err!("unable to access '{}' - {}", filename, err))?;
|
2019-05-30 10:19:23 +00:00
|
|
|
let file_type = metadata.file_type();
|
2019-01-17 10:38:22 +00:00
|
|
|
|
2020-05-30 08:54:38 +00:00
|
|
|
match spec.spec_type {
|
|
|
|
BackupSpecificationType::PXAR => {
|
2019-06-03 08:39:44 +00:00
|
|
|
if !file_type.is_dir() {
|
|
|
|
bail!("got unexpected file type (expected directory)");
|
|
|
|
}
|
2022-02-22 10:03:13 +00:00
|
|
|
upload_list.push((
|
|
|
|
BackupSpecificationType::PXAR,
|
|
|
|
filename.to_owned(),
|
|
|
|
format!("{}.didx", target),
|
|
|
|
0,
|
|
|
|
));
|
2019-06-03 08:39:44 +00:00
|
|
|
}
|
2020-05-30 08:54:38 +00:00
|
|
|
BackupSpecificationType::IMAGE => {
|
2019-06-03 08:39:44 +00:00
|
|
|
if !(file_type.is_file() || file_type.is_block_device()) {
|
|
|
|
bail!("got unexpected file type (expected file or block device)");
|
|
|
|
}
|
2019-05-30 10:19:23 +00:00
|
|
|
|
2019-08-03 11:05:38 +00:00
|
|
|
let size = image_size(&PathBuf::from(filename))?;
|
2019-01-17 10:38:22 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
if size == 0 {
|
|
|
|
bail!("got zero-sized file '{}'", filename);
|
|
|
|
}
|
2019-02-26 11:27:28 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
upload_list.push((
|
|
|
|
BackupSpecificationType::IMAGE,
|
|
|
|
filename.to_owned(),
|
|
|
|
format!("{}.fidx", target),
|
|
|
|
size,
|
|
|
|
));
|
2019-06-03 08:39:44 +00:00
|
|
|
}
|
2020-05-30 08:54:38 +00:00
|
|
|
BackupSpecificationType::CONFIG => {
|
2019-06-03 08:39:44 +00:00
|
|
|
if !file_type.is_file() {
|
|
|
|
bail!("got unexpected file type (expected regular file)");
|
|
|
|
}
|
2022-02-22 10:03:13 +00:00
|
|
|
upload_list.push((
|
|
|
|
BackupSpecificationType::CONFIG,
|
|
|
|
filename.to_owned(),
|
|
|
|
format!("{}.blob", target),
|
|
|
|
metadata.len(),
|
|
|
|
));
|
2019-06-03 08:39:44 +00:00
|
|
|
}
|
2020-05-30 08:54:38 +00:00
|
|
|
BackupSpecificationType::LOGFILE => {
|
2019-07-24 11:07:02 +00:00
|
|
|
if !file_type.is_file() {
|
|
|
|
bail!("got unexpected file type (expected regular file)");
|
|
|
|
}
|
2022-02-22 10:03:13 +00:00
|
|
|
upload_list.push((
|
|
|
|
BackupSpecificationType::LOGFILE,
|
|
|
|
filename.to_owned(),
|
|
|
|
format!("{}.blob", target),
|
|
|
|
metadata.len(),
|
|
|
|
));
|
2019-06-03 08:39:44 +00:00
|
|
|
}
|
2019-02-26 11:27:28 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-15 14:21:34 +00:00
|
|
|
let backup_time = backup_time_opt.unwrap_or_else(epoch_i64);
|
2019-02-26 11:27:28 +00:00
|
|
|
|
2021-11-22 05:26:55 +00:00
|
|
|
let client = connect_rate_limited(&repo, rate_limit)?;
|
2019-03-13 08:47:12 +00:00
|
|
|
record_repository(&repo);
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
println!(
|
|
|
|
"Starting backup: {}/{}/{}",
|
|
|
|
backup_type,
|
|
|
|
backup_id,
|
|
|
|
BackupDir::backup_time_to_string(backup_time)?
|
|
|
|
);
|
2019-07-25 11:44:01 +00:00
|
|
|
|
2021-11-23 16:57:00 +00:00
|
|
|
println!("Client name: {}", proxmox_sys::nodename());
|
2019-07-25 11:44:01 +00:00
|
|
|
|
2020-09-12 13:10:47 +00:00
|
|
|
let start_time = std::time::Instant::now();
|
2019-07-25 11:44:01 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
println!(
|
|
|
|
"Starting backup protocol: {}",
|
|
|
|
strftime_local("%c", epoch_i64())?
|
|
|
|
);
|
2019-03-01 05:48:41 +00:00
|
|
|
|
2021-02-05 15:35:31 +00:00
|
|
|
let (crypt_config, rsa_encrypted_key) = match crypto.enc_key {
|
2019-06-26 05:32:34 +00:00
|
|
|
None => (None, None),
|
2021-02-05 15:35:35 +00:00
|
|
|
Some(key_with_source) => {
|
|
|
|
println!(
|
|
|
|
"{}",
|
|
|
|
format_key_source(&key_with_source.source, "encryption")
|
|
|
|
);
|
|
|
|
|
|
|
|
let (key, created, fingerprint) =
|
2021-03-31 10:21:47 +00:00
|
|
|
decrypt_key(&key_with_source.key, &get_encryption_key_password)?;
|
2020-11-20 16:38:33 +00:00
|
|
|
println!("Encryption key fingerprint: {}", fingerprint);
|
2019-06-26 05:32:34 +00:00
|
|
|
|
2021-01-15 13:38:27 +00:00
|
|
|
let crypt_config = CryptConfig::new(key)?;
|
2019-06-26 05:32:34 +00:00
|
|
|
|
2021-02-05 15:35:32 +00:00
|
|
|
match crypto.master_pubkey {
|
2021-02-05 15:35:35 +00:00
|
|
|
Some(pem_with_source) => {
|
|
|
|
println!("{}", format_key_source(&pem_with_source.source, "master"));
|
|
|
|
|
|
|
|
let rsa = openssl::rsa::Rsa::public_key_from_pem(&pem_with_source.key)?;
|
2021-01-19 11:35:15 +00:00
|
|
|
|
2021-01-21 10:56:54 +00:00
|
|
|
let mut key_config = KeyConfig::without_password(key)?;
|
2021-01-19 11:35:15 +00:00
|
|
|
key_config.created = created; // keep original value
|
|
|
|
|
2020-12-16 13:41:07 +00:00
|
|
|
let enc_key = rsa_encrypt_key_config(rsa, &key_config)?;
|
2020-11-20 16:38:33 +00:00
|
|
|
|
2020-07-08 08:56:16 +00:00
|
|
|
(Some(Arc::new(crypt_config)), Some(enc_key))
|
2022-02-22 10:03:13 +00:00
|
|
|
}
|
2020-07-08 08:56:16 +00:00
|
|
|
_ => (Some(Arc::new(crypt_config)), None),
|
2019-06-26 05:32:34 +00:00
|
|
|
}
|
2019-06-19 15:16:41 +00:00
|
|
|
}
|
|
|
|
};
|
2019-06-13 09:47:23 +00:00
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
let client = BackupWriter::start(
|
|
|
|
client,
|
2020-06-25 10:23:30 +00:00
|
|
|
crypt_config.clone(),
|
2019-12-16 09:06:26 +00:00
|
|
|
repo.store(),
|
|
|
|
backup_type,
|
2021-12-30 11:57:37 +00:00
|
|
|
backup_id,
|
2019-12-16 09:06:26 +00:00
|
|
|
backup_time,
|
|
|
|
verbose,
|
2022-02-22 10:03:13 +00:00
|
|
|
false,
|
|
|
|
)
|
|
|
|
.await?;
|
2019-12-16 09:06:26 +00:00
|
|
|
|
2020-11-20 16:38:40 +00:00
|
|
|
let download_previous_manifest = match client.previous_backup_time().await {
|
|
|
|
Ok(Some(backup_time)) => {
|
|
|
|
println!(
|
|
|
|
"Downloading previous manifest ({})",
|
|
|
|
strftime_local("%c", backup_time)?
|
|
|
|
);
|
|
|
|
true
|
|
|
|
}
|
|
|
|
Ok(None) => {
|
|
|
|
println!("No previous manifest available.");
|
|
|
|
false
|
|
|
|
}
|
|
|
|
Err(_) => {
|
|
|
|
// Fallback for outdated server, TODO remove/bubble up with 2.0
|
|
|
|
true
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
let previous_manifest = if download_previous_manifest {
|
|
|
|
match client.download_previous_manifest().await {
|
|
|
|
Ok(previous_manifest) => {
|
|
|
|
match previous_manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref)) {
|
|
|
|
Ok(()) => Some(Arc::new(previous_manifest)),
|
|
|
|
Err(err) => {
|
|
|
|
println!("Couldn't re-use previous manifest - {}", err);
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
2020-11-20 16:38:37 +00:00
|
|
|
}
|
2020-11-20 16:38:40 +00:00
|
|
|
Err(err) => {
|
|
|
|
println!("Couldn't download previous manifest - {}", err);
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
None
|
2020-06-25 10:23:30 +00:00
|
|
|
};
|
|
|
|
|
2020-09-12 13:10:47 +00:00
|
|
|
let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
|
2019-12-16 09:06:26 +00:00
|
|
|
let mut manifest = BackupManifest::new(snapshot);
|
|
|
|
|
2020-06-12 08:04:59 +00:00
|
|
|
let mut catalog = None;
|
2021-01-25 13:42:48 +00:00
|
|
|
let mut catalog_result_rx = None;
|
2019-12-16 09:06:26 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let log_file = |desc: &str, file: &str, target: &str| {
|
|
|
|
let what = if dry_run { "Would upload" } else { "Upload" };
|
|
|
|
println!("{} {} '{}' to '{}' as {}", what, desc, file, repo, target);
|
2022-02-18 09:55:38 +00:00
|
|
|
};
|
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
for (backup_type, filename, target, size) in upload_list {
|
2022-02-18 09:55:38 +00:00
|
|
|
match (backup_type, dry_run) {
|
2022-02-18 14:02:23 +00:00
|
|
|
// dry-run
|
|
|
|
(BackupSpecificationType::CONFIG, true) => log_file("config file", &filename, &target),
|
|
|
|
(BackupSpecificationType::LOGFILE, true) => log_file("log file", &filename, &target),
|
|
|
|
(BackupSpecificationType::PXAR, true) => log_file("directory", &filename, &target),
|
|
|
|
(BackupSpecificationType::IMAGE, true) => log_file("image", &filename, &target),
|
|
|
|
// no dry-run
|
2022-02-18 09:55:38 +00:00
|
|
|
(BackupSpecificationType::CONFIG, false) => {
|
2021-01-25 13:42:52 +00:00
|
|
|
let upload_options = UploadOptions {
|
|
|
|
compress: true,
|
2021-02-05 15:35:31 +00:00
|
|
|
encrypt: crypto.mode == CryptMode::Encrypt,
|
2021-01-25 13:42:52 +00:00
|
|
|
..UploadOptions::default()
|
|
|
|
};
|
|
|
|
|
2022-02-18 14:02:23 +00:00
|
|
|
log_file("config file", &filename, &target);
|
2019-12-16 09:06:26 +00:00
|
|
|
let stats = client
|
2021-01-25 13:42:52 +00:00
|
|
|
.upload_blob_from_file(&filename, &target, upload_options)
|
2019-12-16 09:06:26 +00:00
|
|
|
.await?;
|
2021-02-05 15:35:31 +00:00
|
|
|
manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
|
2019-12-16 09:06:26 +00:00
|
|
|
}
|
2022-02-22 10:03:13 +00:00
|
|
|
(BackupSpecificationType::LOGFILE, false) => {
|
|
|
|
// fixme: remove - not needed anymore ?
|
2021-01-25 13:42:52 +00:00
|
|
|
let upload_options = UploadOptions {
|
|
|
|
compress: true,
|
2021-02-05 15:35:31 +00:00
|
|
|
encrypt: crypto.mode == CryptMode::Encrypt,
|
2021-01-25 13:42:52 +00:00
|
|
|
..UploadOptions::default()
|
|
|
|
};
|
|
|
|
|
2022-02-18 14:02:23 +00:00
|
|
|
log_file("log file", &filename, &target);
|
2019-12-16 09:06:26 +00:00
|
|
|
let stats = client
|
2021-01-25 13:42:52 +00:00
|
|
|
.upload_blob_from_file(&filename, &target, upload_options)
|
2019-12-16 09:06:26 +00:00
|
|
|
.await?;
|
2021-02-05 15:35:31 +00:00
|
|
|
manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
|
2019-12-16 09:06:26 +00:00
|
|
|
}
|
2022-02-18 09:55:38 +00:00
|
|
|
(BackupSpecificationType::PXAR, false) => {
|
2020-06-12 08:04:59 +00:00
|
|
|
// start catalog upload on first use
|
|
|
|
if catalog.is_none() {
|
2022-02-22 10:03:13 +00:00
|
|
|
let catalog_upload_res =
|
|
|
|
spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?;
|
2021-01-25 13:42:48 +00:00
|
|
|
catalog = Some(catalog_upload_res.catalog_writer);
|
|
|
|
catalog_result_rx = Some(catalog_upload_res.result);
|
2020-06-12 08:04:59 +00:00
|
|
|
}
|
|
|
|
let catalog = catalog.as_ref().unwrap();
|
|
|
|
|
2022-02-18 14:02:23 +00:00
|
|
|
log_file("directory", &filename, &target);
|
2022-02-22 10:03:13 +00:00
|
|
|
catalog
|
|
|
|
.lock()
|
|
|
|
.unwrap()
|
|
|
|
.start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
|
2021-01-25 13:42:54 +00:00
|
|
|
|
2021-07-19 08:50:18 +00:00
|
|
|
let pxar_options = pbs_client::pxar::PxarCreateOptions {
|
2021-01-25 13:42:54 +00:00
|
|
|
device_set: devices.clone(),
|
|
|
|
patterns: pattern_list.clone(),
|
|
|
|
entries_max: entries_max as usize,
|
|
|
|
skip_lost_and_found,
|
|
|
|
verbose,
|
|
|
|
};
|
|
|
|
|
2021-01-25 13:42:52 +00:00
|
|
|
let upload_options = UploadOptions {
|
|
|
|
previous_manifest: previous_manifest.clone(),
|
|
|
|
compress: true,
|
2021-02-05 15:35:31 +00:00
|
|
|
encrypt: crypto.mode == CryptMode::Encrypt,
|
2021-01-25 13:42:52 +00:00
|
|
|
..UploadOptions::default()
|
|
|
|
};
|
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
let stats = backup_directory(
|
|
|
|
&client,
|
|
|
|
&filename,
|
|
|
|
&target,
|
|
|
|
chunk_size_opt,
|
|
|
|
catalog.clone(),
|
2021-01-25 13:42:54 +00:00
|
|
|
pxar_options,
|
2021-01-25 13:42:52 +00:00
|
|
|
upload_options,
|
2022-02-22 10:03:13 +00:00
|
|
|
)
|
|
|
|
.await?;
|
2021-02-05 15:35:31 +00:00
|
|
|
manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
|
2019-12-16 09:06:26 +00:00
|
|
|
catalog.lock().unwrap().end_directory()?;
|
|
|
|
}
|
2022-02-18 09:55:38 +00:00
|
|
|
(BackupSpecificationType::IMAGE, false) => {
|
2022-02-18 14:02:23 +00:00
|
|
|
log_file("image", &filename, &target);
|
2021-01-25 13:42:52 +00:00
|
|
|
|
|
|
|
let upload_options = UploadOptions {
|
|
|
|
previous_manifest: previous_manifest.clone(),
|
|
|
|
fixed_size: Some(size),
|
|
|
|
compress: true,
|
2021-02-05 15:35:31 +00:00
|
|
|
encrypt: crypto.mode == CryptMode::Encrypt,
|
2021-01-25 13:42:52 +00:00
|
|
|
};
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let stats =
|
|
|
|
backup_image(&client, &filename, &target, chunk_size_opt, upload_options)
|
|
|
|
.await?;
|
2021-02-05 15:35:31 +00:00
|
|
|
manifest.add_file(target, stats.size, stats.csum, crypto.mode)?;
|
2019-05-30 10:46:01 +00:00
|
|
|
}
|
|
|
|
}
|
2019-12-16 09:06:26 +00:00
|
|
|
}
|
2018-12-16 13:44:44 +00:00
|
|
|
|
2022-02-18 09:55:38 +00:00
|
|
|
if dry_run {
|
|
|
|
println!("dry-run: no upload happend");
|
|
|
|
return Ok(Value::Null);
|
|
|
|
}
|
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
// finalize and upload catalog
|
2020-06-12 08:04:59 +00:00
|
|
|
if let Some(catalog) = catalog {
|
2019-12-16 09:06:26 +00:00
|
|
|
let mutex = Arc::try_unwrap(catalog)
|
|
|
|
.map_err(|_| format_err!("unable to get catalog (still used)"))?;
|
|
|
|
let mut catalog = mutex.into_inner().unwrap();
|
2019-11-08 09:35:48 +00:00
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
catalog.finish()?;
|
2019-08-09 07:46:49 +00:00
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
drop(catalog); // close upload stream
|
2019-08-09 07:46:49 +00:00
|
|
|
|
2021-01-25 13:42:48 +00:00
|
|
|
if let Some(catalog_result_rx) = catalog_result_rx {
|
2020-06-12 08:04:59 +00:00
|
|
|
let stats = catalog_result_rx.await??;
|
2021-02-05 15:35:31 +00:00
|
|
|
manifest.add_file(CATALOG_NAME.to_owned(), stats.size, stats.csum, crypto.mode)?;
|
2020-06-12 08:04:59 +00:00
|
|
|
}
|
2019-12-16 09:06:26 +00:00
|
|
|
}
|
2019-08-09 07:46:49 +00:00
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
if let Some(rsa_encrypted_key) = rsa_encrypted_key {
|
2020-12-16 13:41:05 +00:00
|
|
|
let target = ENCRYPTED_KEY_BLOB_NAME;
|
2019-12-16 09:06:26 +00:00
|
|
|
println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
|
2022-02-22 10:03:13 +00:00
|
|
|
let options = UploadOptions {
|
|
|
|
compress: false,
|
|
|
|
encrypt: false,
|
|
|
|
..UploadOptions::default()
|
|
|
|
};
|
2019-12-16 09:06:26 +00:00
|
|
|
let stats = client
|
2021-01-25 13:42:52 +00:00
|
|
|
.upload_blob_from_data(rsa_encrypted_key, target, options)
|
2019-12-16 09:06:26 +00:00
|
|
|
.await?;
|
2021-02-05 15:35:31 +00:00
|
|
|
manifest.add_file(target.to_string(), stats.size, stats.csum, crypto.mode)?;
|
2019-12-16 09:06:26 +00:00
|
|
|
}
|
|
|
|
// create manifest (index.json)
|
2020-07-08 12:06:50 +00:00
|
|
|
// manifests are never encrypted, but include a signature
|
2022-02-22 10:03:13 +00:00
|
|
|
let manifest = manifest
|
|
|
|
.to_string(crypt_config.as_ref().map(Arc::as_ref))
|
2020-07-09 07:20:49 +00:00
|
|
|
.map_err(|err| format_err!("unable to format manifest - {}", err))?;
|
2020-07-08 12:06:50 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
if verbose {
|
|
|
|
println!("Upload index.json to '{}'", repo)
|
|
|
|
};
|
|
|
|
let options = UploadOptions {
|
|
|
|
compress: true,
|
|
|
|
encrypt: false,
|
|
|
|
..UploadOptions::default()
|
|
|
|
};
|
2019-12-16 09:06:26 +00:00
|
|
|
client
|
2021-01-25 13:42:52 +00:00
|
|
|
.upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, options)
|
2019-12-16 09:06:26 +00:00
|
|
|
.await?;
|
2019-08-01 10:39:02 +00:00
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
client.finish().await?;
|
2019-05-28 08:12:44 +00:00
|
|
|
|
2020-09-12 13:10:47 +00:00
|
|
|
let end_time = std::time::Instant::now();
|
|
|
|
let elapsed = end_time.duration_since(start_time);
|
|
|
|
println!("Duration: {:.2}s", elapsed.as_secs_f64());
|
2019-03-05 07:11:40 +00:00
|
|
|
|
2020-09-12 13:10:47 +00:00
|
|
|
println!("End Time: {}", strftime_local("%c", epoch_i64())?);
|
2018-12-18 10:06:03 +00:00
|
|
|
|
2019-12-16 09:06:26 +00:00
|
|
|
Ok(Value::Null)
|
2019-02-27 07:38:32 +00:00
|
|
|
}
|
|
|
|
|
2020-06-12 09:40:18 +00:00
|
|
|
async fn dump_image<W: Write>(
|
2019-10-05 09:48:51 +00:00
|
|
|
client: Arc<BackupReader>,
|
|
|
|
crypt_config: Option<Arc<CryptConfig>>,
|
2020-08-10 11:25:07 +00:00
|
|
|
crypt_mode: CryptMode,
|
2019-10-05 09:48:51 +00:00
|
|
|
index: FixedIndexReader,
|
|
|
|
mut writer: W,
|
2019-10-08 11:04:10 +00:00
|
|
|
verbose: bool,
|
2019-10-05 09:48:51 +00:00
|
|
|
) -> Result<(), Error> {
|
|
|
|
let most_used = index.find_most_used_chunks(8);
|
|
|
|
|
2020-08-10 11:25:07 +00:00
|
|
|
let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, crypt_mode, most_used);
|
2019-10-05 09:48:51 +00:00
|
|
|
|
|
|
|
// Note: we avoid using BufferedFixedReader, because that add an additional buffer/copy
|
|
|
|
// and thus slows down reading. Instead, directly use RemoteChunkReader
|
2019-10-08 11:04:10 +00:00
|
|
|
let mut per = 0;
|
|
|
|
let mut bytes = 0;
|
|
|
|
let start_time = std::time::Instant::now();
|
|
|
|
|
2019-10-05 09:48:51 +00:00
|
|
|
for pos in 0..index.index_count() {
|
|
|
|
let digest = index.index_digest(pos).unwrap();
|
2021-12-30 11:57:37 +00:00
|
|
|
let raw_data = chunk_reader.read_chunk(digest).await?;
|
2019-10-05 09:48:51 +00:00
|
|
|
writer.write_all(&raw_data)?;
|
2019-10-08 11:04:10 +00:00
|
|
|
bytes += raw_data.len();
|
|
|
|
if verbose {
|
2022-02-22 10:03:13 +00:00
|
|
|
let next_per = ((pos + 1) * 100) / index.index_count();
|
2019-10-08 11:04:10 +00:00
|
|
|
if per != next_per {
|
2022-02-22 10:03:13 +00:00
|
|
|
eprintln!(
|
|
|
|
"progress {}% (read {} bytes, duration {} sec)",
|
|
|
|
next_per,
|
|
|
|
bytes,
|
|
|
|
start_time.elapsed().as_secs()
|
|
|
|
);
|
2019-10-08 11:04:10 +00:00
|
|
|
per = next_per;
|
|
|
|
}
|
|
|
|
}
|
2019-10-05 09:48:51 +00:00
|
|
|
}
|
|
|
|
|
2019-10-08 11:04:10 +00:00
|
|
|
let end_time = std::time::Instant::now();
|
|
|
|
let elapsed = end_time.duration_since(start_time);
|
2022-02-22 10:03:13 +00:00
|
|
|
eprintln!(
|
|
|
|
"restore image complete (bytes={}, duration={:.2}s, speed={:.2}MB/s)",
|
|
|
|
bytes,
|
|
|
|
elapsed.as_secs_f64(),
|
|
|
|
bytes as f64 / (1024.0 * 1024.0 * elapsed.as_secs_f64())
|
2019-10-08 11:04:10 +00:00
|
|
|
);
|
|
|
|
|
2019-10-05 09:48:51 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2020-06-02 16:41:31 +00:00
|
|
|
fn parse_archive_type(name: &str) -> (String, ArchiveType) {
|
2020-06-02 16:41:32 +00:00
|
|
|
if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
|
|
|
|
(name.into(), archive_type(name).unwrap())
|
|
|
|
} else if name.ends_with(".pxar") {
|
2020-06-02 16:41:31 +00:00
|
|
|
(format!("{}.didx", name), ArchiveType::DynamicIndex)
|
|
|
|
} else if name.ends_with(".img") {
|
|
|
|
(format!("{}.fidx", name), ArchiveType::FixedIndex)
|
|
|
|
} else {
|
|
|
|
(format!("{}.blob", name), ArchiveType::Blob)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-12-16 12:34:49 +00:00
|
|
|
#[api(
|
|
|
|
input: {
|
|
|
|
properties: {
|
|
|
|
repository: {
|
|
|
|
schema: REPO_URL_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
snapshot: {
|
|
|
|
type: String,
|
|
|
|
description: "Group/Snapshot path.",
|
|
|
|
},
|
|
|
|
"archive-name": {
|
|
|
|
description: "Backup archive name.",
|
|
|
|
type: String,
|
|
|
|
},
|
|
|
|
target: {
|
|
|
|
type: String,
|
2020-01-24 12:31:46 +00:00
|
|
|
description: r###"Target directory path. Use '-' to write to standard output.
|
2019-12-16 09:06:26 +00:00
|
|
|
|
2021-03-10 15:37:09 +00:00
|
|
|
We do not extract '.pxar' archives when writing to standard output.
|
2019-12-16 09:06:26 +00:00
|
|
|
|
2019-12-16 12:34:49 +00:00
|
|
|
"###
|
|
|
|
},
|
2021-11-06 17:46:58 +00:00
|
|
|
rate: {
|
|
|
|
schema: TRAFFIC_CONTROL_RATE_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
burst: {
|
|
|
|
schema: TRAFFIC_CONTROL_BURST_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
2019-12-16 12:34:49 +00:00
|
|
|
"allow-existing-dirs": {
|
|
|
|
type: Boolean,
|
|
|
|
description: "Do not fail if directories already exists.",
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
keyfile: {
|
|
|
|
schema: KEYFILE_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
2020-07-08 11:52:17 +00:00
|
|
|
"keyfd": {
|
|
|
|
schema: KEYFD_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
2020-07-08 08:09:15 +00:00
|
|
|
"crypt-mode": {
|
|
|
|
type: CryptMode,
|
2020-07-06 12:35:28 +00:00
|
|
|
optional: true,
|
|
|
|
},
|
2019-12-16 12:34:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
)]
|
|
|
|
/// Restore backup repository.
|
|
|
|
async fn restore(param: Value) -> Result<Value, Error> {
|
2019-07-16 05:30:04 +00:00
|
|
|
let repo = extract_repository_from_value(¶m)?;
|
2019-03-06 09:50:46 +00:00
|
|
|
|
2019-07-03 13:45:12 +00:00
|
|
|
let verbose = param["verbose"].as_bool().unwrap_or(false);
|
|
|
|
|
2019-07-29 10:49:15 +00:00
|
|
|
let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
|
|
|
|
|
2021-07-20 09:06:53 +00:00
|
|
|
let archive_name = json::required_string_param(¶m, "archive-name")?;
|
2019-03-11 13:31:01 +00:00
|
|
|
|
2021-11-22 05:26:55 +00:00
|
|
|
let rate = match param["rate"].as_str() {
|
|
|
|
Some(s) => Some(s.parse::<HumanByte>()?),
|
|
|
|
None => None,
|
|
|
|
};
|
|
|
|
let burst = match param["burst"].as_str() {
|
|
|
|
Some(s) => Some(s.parse::<HumanByte>()?),
|
|
|
|
None => None,
|
|
|
|
};
|
|
|
|
|
|
|
|
let rate_limit = RateLimitConfig::with_same_inout(rate, burst);
|
2019-03-13 08:47:12 +00:00
|
|
|
|
2021-11-22 05:26:55 +00:00
|
|
|
let client = connect_rate_limited(&repo, rate_limit)?;
|
2019-03-13 08:47:12 +00:00
|
|
|
record_repository(&repo);
|
2019-03-11 13:31:01 +00:00
|
|
|
|
2021-07-20 09:06:53 +00:00
|
|
|
let path = json::required_string_param(¶m, "snapshot")?;
|
2019-03-06 09:50:46 +00:00
|
|
|
|
2019-07-03 13:45:12 +00:00
|
|
|
let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
|
2020-06-23 06:16:56 +00:00
|
|
|
let group: BackupGroup = path.parse()?;
|
2020-01-07 14:18:36 +00:00
|
|
|
api_datastore_latest_snapshot(&client, repo.store(), group).await?
|
2019-03-11 13:31:01 +00:00
|
|
|
} else {
|
2020-06-23 06:09:52 +00:00
|
|
|
let snapshot: BackupDir = path.parse()?;
|
2022-02-22 10:03:13 +00:00
|
|
|
(
|
|
|
|
snapshot.group().backup_type().to_owned(),
|
|
|
|
snapshot.group().backup_id().to_owned(),
|
|
|
|
snapshot.backup_time(),
|
|
|
|
)
|
2019-07-03 13:45:12 +00:00
|
|
|
};
|
2019-03-06 09:50:46 +00:00
|
|
|
|
2021-07-20 09:06:53 +00:00
|
|
|
let target = json::required_string_param(¶m, "target")?;
|
2019-07-05 10:14:50 +00:00
|
|
|
let target = if target == "-" { None } else { Some(target) };
|
2019-03-06 10:18:46 +00:00
|
|
|
|
2021-02-05 15:35:31 +00:00
|
|
|
let crypto = crypto_parameters(¶m)?;
|
2019-03-06 10:18:46 +00:00
|
|
|
|
2021-02-05 15:35:31 +00:00
|
|
|
let crypt_config = match crypto.enc_key {
|
2019-07-03 13:45:12 +00:00
|
|
|
None => None,
|
2021-02-05 15:35:35 +00:00
|
|
|
Some(ref key) => {
|
|
|
|
let (key, _, _) =
|
2021-03-31 10:21:47 +00:00
|
|
|
decrypt_key(&key.key, &get_encryption_key_password).map_err(|err| {
|
2021-02-05 15:35:35 +00:00
|
|
|
eprintln!("{}", format_key_source(&key.source, "encryption"));
|
|
|
|
err
|
|
|
|
})?;
|
2019-07-03 13:45:12 +00:00
|
|
|
Some(Arc::new(CryptConfig::new(key)?))
|
|
|
|
}
|
|
|
|
};
|
2019-03-11 13:31:01 +00:00
|
|
|
|
2019-10-12 13:50:26 +00:00
|
|
|
let client = BackupReader::start(
|
|
|
|
client,
|
|
|
|
crypt_config.clone(),
|
|
|
|
repo.store(),
|
|
|
|
&backup_type,
|
|
|
|
&backup_id,
|
|
|
|
backup_time,
|
|
|
|
true,
|
2022-02-22 10:03:13 +00:00
|
|
|
)
|
|
|
|
.await?;
|
2019-07-03 13:45:12 +00:00
|
|
|
|
2020-12-16 13:41:06 +00:00
|
|
|
let (archive_name, archive_type) = parse_archive_type(archive_name);
|
|
|
|
|
2020-07-08 14:07:14 +00:00
|
|
|
let (manifest, backup_index_data) = client.download_manifest().await?;
|
2019-09-02 12:14:32 +00:00
|
|
|
|
2020-12-16 13:41:06 +00:00
|
|
|
if archive_name == ENCRYPTED_KEY_BLOB_NAME && crypt_config.is_none() {
|
|
|
|
eprintln!("Restoring encrypted key blob without original key - skipping manifest fingerprint check!")
|
|
|
|
} else {
|
2021-02-05 15:35:35 +00:00
|
|
|
if manifest.signature.is_some() {
|
|
|
|
if let Some(key) = &crypto.enc_key {
|
|
|
|
eprintln!("{}", format_key_source(&key.source, "encryption"));
|
|
|
|
}
|
|
|
|
if let Some(config) = &crypt_config {
|
2021-09-07 07:22:14 +00:00
|
|
|
eprintln!("Fingerprint: {}", Fingerprint::new(config.fingerprint()));
|
2021-02-05 15:35:35 +00:00
|
|
|
}
|
|
|
|
}
|
2020-12-16 13:41:06 +00:00
|
|
|
manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
|
|
|
|
}
|
2020-06-02 16:41:31 +00:00
|
|
|
|
|
|
|
if archive_name == MANIFEST_BLOB_NAME {
|
2019-09-02 12:14:32 +00:00
|
|
|
if let Some(target) = target {
|
2021-10-20 12:56:15 +00:00
|
|
|
replace_file(target, &backup_index_data, CreateOptions::new(), false)?;
|
2019-09-02 12:14:32 +00:00
|
|
|
} else {
|
|
|
|
let stdout = std::io::stdout();
|
|
|
|
let mut writer = stdout.lock();
|
2022-02-22 10:03:13 +00:00
|
|
|
writer
|
|
|
|
.write_all(&backup_index_data)
|
2019-09-02 12:14:32 +00:00
|
|
|
.map_err(|err| format_err!("unable to pipe data - {}", err))?;
|
|
|
|
}
|
|
|
|
|
2020-08-10 11:25:07 +00:00
|
|
|
return Ok(Value::Null);
|
|
|
|
}
|
|
|
|
|
|
|
|
let file_info = manifest.lookup_file_info(&archive_name)?;
|
|
|
|
|
|
|
|
if archive_type == ArchiveType::Blob {
|
2020-06-02 16:41:31 +00:00
|
|
|
let mut reader = client.download_blob(&manifest, &archive_name).await?;
|
2019-07-05 09:36:45 +00:00
|
|
|
|
2019-07-05 10:14:50 +00:00
|
|
|
if let Some(target) = target {
|
2022-02-22 10:03:13 +00:00
|
|
|
let mut writer = std::fs::OpenOptions::new()
|
2019-09-02 12:13:31 +00:00
|
|
|
.write(true)
|
|
|
|
.create(true)
|
|
|
|
.create_new(true)
|
|
|
|
.open(target)
|
2022-02-22 10:03:13 +00:00
|
|
|
.map_err(|err| {
|
|
|
|
format_err!("unable to create target file {:?} - {}", target, err)
|
|
|
|
})?;
|
2019-09-02 12:13:31 +00:00
|
|
|
std::io::copy(&mut reader, &mut writer)?;
|
2019-07-05 10:14:50 +00:00
|
|
|
} else {
|
|
|
|
let stdout = std::io::stdout();
|
|
|
|
let mut writer = stdout.lock();
|
2019-09-02 12:13:31 +00:00
|
|
|
std::io::copy(&mut reader, &mut writer)
|
2019-07-05 10:14:50 +00:00
|
|
|
.map_err(|err| format_err!("unable to pipe data - {}", err))?;
|
|
|
|
}
|
2020-06-02 16:41:31 +00:00
|
|
|
} else if archive_type == ArchiveType::DynamicIndex {
|
2022-02-22 10:03:13 +00:00
|
|
|
let index = client
|
|
|
|
.download_dynamic_index(&manifest, &archive_name)
|
|
|
|
.await?;
|
2019-09-03 11:12:16 +00:00
|
|
|
|
2019-07-05 08:42:46 +00:00
|
|
|
let most_used = index.find_most_used_chunks(8);
|
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let chunk_reader = RemoteChunkReader::new(
|
|
|
|
client.clone(),
|
|
|
|
crypt_config,
|
|
|
|
file_info.chunk_crypt_mode(),
|
|
|
|
most_used,
|
|
|
|
);
|
2019-07-05 08:42:46 +00:00
|
|
|
|
2019-07-05 07:19:56 +00:00
|
|
|
let mut reader = BufferedDynamicReader::new(index, chunk_reader);
|
2019-07-03 13:45:12 +00:00
|
|
|
|
2021-07-19 08:50:18 +00:00
|
|
|
let options = pbs_client::pxar::PxarExtractOptions {
|
2021-01-25 13:42:55 +00:00
|
|
|
match_list: &[],
|
|
|
|
extract_match_default: true,
|
|
|
|
allow_existing_dirs,
|
|
|
|
on_error: None,
|
|
|
|
};
|
|
|
|
|
2019-07-05 10:14:50 +00:00
|
|
|
if let Some(target) = target {
|
2021-07-19 08:50:18 +00:00
|
|
|
pbs_client::pxar::extract_archive(
|
2020-03-23 14:03:18 +00:00
|
|
|
pxar::decoder::Decoder::from_std(reader)?,
|
|
|
|
Path::new(target),
|
2021-07-19 08:50:18 +00:00
|
|
|
pbs_client::pxar::Flags::DEFAULT,
|
2020-03-23 14:03:18 +00:00
|
|
|
|path| {
|
|
|
|
if verbose {
|
|
|
|
println!("{:?}", path);
|
|
|
|
}
|
|
|
|
},
|
2021-01-25 13:42:55 +00:00
|
|
|
options,
|
2020-03-23 14:03:18 +00:00
|
|
|
)
|
|
|
|
.map_err(|err| format_err!("error extracting archive - {}", err))?;
|
2019-07-05 10:14:50 +00:00
|
|
|
} else {
|
2019-10-05 09:48:51 +00:00
|
|
|
let mut writer = std::fs::OpenOptions::new()
|
|
|
|
.write(true)
|
|
|
|
.open("/dev/stdout")
|
|
|
|
.map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?;
|
2019-07-05 07:19:56 +00:00
|
|
|
|
2019-07-05 10:14:50 +00:00
|
|
|
std::io::copy(&mut reader, &mut writer)
|
|
|
|
.map_err(|err| format_err!("unable to pipe data - {}", err))?;
|
|
|
|
}
|
2020-06-02 16:41:31 +00:00
|
|
|
} else if archive_type == ArchiveType::FixedIndex {
|
2022-02-22 10:03:13 +00:00
|
|
|
let index = client
|
|
|
|
.download_fixed_index(&manifest, &archive_name)
|
|
|
|
.await?;
|
2019-09-03 11:12:16 +00:00
|
|
|
|
2019-10-05 09:48:51 +00:00
|
|
|
let mut writer = if let Some(target) = target {
|
|
|
|
std::fs::OpenOptions::new()
|
2019-07-05 10:14:50 +00:00
|
|
|
.write(true)
|
|
|
|
.create(true)
|
|
|
|
.create_new(true)
|
|
|
|
.open(target)
|
2019-10-05 09:48:51 +00:00
|
|
|
.map_err(|err| format_err!("unable to create target file {:?} - {}", target, err))?
|
2019-07-05 10:14:50 +00:00
|
|
|
} else {
|
2019-10-05 09:48:51 +00:00
|
|
|
std::fs::OpenOptions::new()
|
|
|
|
.write(true)
|
|
|
|
.open("/dev/stdout")
|
|
|
|
.map_err(|err| format_err!("unable to open /dev/stdout - {}", err))?
|
|
|
|
};
|
2019-07-05 07:19:56 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
dump_image(
|
|
|
|
client.clone(),
|
|
|
|
crypt_config.clone(),
|
|
|
|
file_info.chunk_crypt_mode(),
|
|
|
|
index,
|
|
|
|
&mut writer,
|
|
|
|
verbose,
|
|
|
|
)
|
|
|
|
.await?;
|
2019-06-25 11:00:37 +00:00
|
|
|
}
|
2019-06-26 07:18:59 +00:00
|
|
|
|
|
|
|
Ok(Value::Null)
|
2019-06-25 09:17:24 +00:00
|
|
|
}
|
|
|
|
|
2021-07-16 08:53:20 +00:00
|
|
|
#[api(
|
|
|
|
input: {
|
|
|
|
properties: {
|
|
|
|
"dry-run": {
|
|
|
|
type: bool,
|
|
|
|
optional: true,
|
|
|
|
description: "Just show what prune would do, but do not delete anything.",
|
|
|
|
},
|
|
|
|
group: {
|
|
|
|
type: String,
|
|
|
|
description: "Backup group",
|
|
|
|
},
|
|
|
|
"prune-options": {
|
|
|
|
type: PruneOptions,
|
|
|
|
flatten: true,
|
|
|
|
},
|
|
|
|
"output-format": {
|
|
|
|
schema: OUTPUT_FORMAT,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
quiet: {
|
|
|
|
type: bool,
|
|
|
|
optional: true,
|
|
|
|
default: false,
|
|
|
|
description: "Minimal output - only show removals.",
|
|
|
|
},
|
|
|
|
repository: {
|
|
|
|
schema: REPO_URL_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
)]
|
|
|
|
/// Prune a backup repository.
|
|
|
|
async fn prune(
|
|
|
|
dry_run: Option<bool>,
|
|
|
|
group: String,
|
|
|
|
prune_options: PruneOptions,
|
|
|
|
quiet: bool,
|
2022-02-22 10:03:13 +00:00
|
|
|
mut param: Value,
|
2021-07-16 08:53:20 +00:00
|
|
|
) -> Result<Value, Error> {
|
2019-07-16 05:30:04 +00:00
|
|
|
let repo = extract_repository_from_value(¶m)?;
|
2019-02-27 15:53:17 +00:00
|
|
|
|
2021-12-04 13:11:55 +00:00
|
|
|
let client = connect(&repo)?;
|
2019-02-27 15:53:17 +00:00
|
|
|
|
2019-03-13 08:47:12 +00:00
|
|
|
let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
|
2019-02-27 15:53:17 +00:00
|
|
|
|
2020-06-23 06:16:56 +00:00
|
|
|
let group: BackupGroup = group.parse()?;
|
2020-02-27 11:41:15 +00:00
|
|
|
|
2021-04-30 12:51:56 +00:00
|
|
|
let output_format = extract_output_format(&mut param);
|
2019-07-27 06:49:14 +00:00
|
|
|
|
2021-07-16 08:53:20 +00:00
|
|
|
let mut api_param = serde_json::to_value(prune_options)?;
|
|
|
|
if let Some(dry_run) = dry_run {
|
|
|
|
api_param["dry-run"] = dry_run.into();
|
|
|
|
}
|
|
|
|
api_param["backup-type"] = group.backup_type().into();
|
|
|
|
api_param["backup-id"] = group.backup_id().into();
|
2019-02-27 15:53:17 +00:00
|
|
|
|
2021-07-16 08:53:20 +00:00
|
|
|
let mut result = client.post(&path, Some(api_param)).await?;
|
2019-12-06 12:35:25 +00:00
|
|
|
|
2019-12-16 10:00:43 +00:00
|
|
|
record_repository(&repo);
|
2019-12-08 09:59:47 +00:00
|
|
|
|
2020-05-05 04:45:37 +00:00
|
|
|
let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
|
|
|
|
let item: PruneListItem = serde_json::from_value(record.to_owned())?;
|
2020-09-11 12:34:38 +00:00
|
|
|
let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
|
2020-05-05 04:45:37 +00:00
|
|
|
Ok(snapshot.relative_path().to_str().unwrap().to_owned())
|
|
|
|
};
|
|
|
|
|
2020-06-03 08:11:37 +00:00
|
|
|
let render_prune_action = |v: &Value, _record: &Value| -> Result<String, Error> {
|
|
|
|
Ok(match v.as_bool() {
|
|
|
|
Some(true) => "keep",
|
|
|
|
Some(false) => "remove",
|
|
|
|
None => "unknown",
|
2022-02-22 10:03:13 +00:00
|
|
|
}
|
|
|
|
.to_string())
|
2020-06-03 08:11:37 +00:00
|
|
|
};
|
|
|
|
|
2020-05-05 04:45:37 +00:00
|
|
|
let options = default_table_format_options()
|
|
|
|
.sortby("backup-type", false)
|
|
|
|
.sortby("backup-id", false)
|
|
|
|
.sortby("backup-time", false)
|
2022-02-22 10:03:13 +00:00
|
|
|
.column(
|
|
|
|
ColumnConfig::new("backup-id")
|
|
|
|
.renderer(render_snapshot_path)
|
|
|
|
.header("snapshot"),
|
|
|
|
)
|
|
|
|
.column(
|
|
|
|
ColumnConfig::new("backup-time")
|
|
|
|
.renderer(pbs_tools::format::render_epoch)
|
|
|
|
.header("date"),
|
|
|
|
)
|
|
|
|
.column(
|
|
|
|
ColumnConfig::new("keep")
|
|
|
|
.renderer(render_prune_action)
|
|
|
|
.header("action"),
|
|
|
|
);
|
2020-05-05 04:45:37 +00:00
|
|
|
|
2021-08-26 09:00:37 +00:00
|
|
|
let return_type = &pbs_api_types::ADMIN_DATASTORE_PRUNE_RETURN_TYPE;
|
2020-05-05 04:45:37 +00:00
|
|
|
|
|
|
|
let mut data = result["data"].take();
|
|
|
|
|
2020-06-03 08:11:37 +00:00
|
|
|
if quiet {
|
2022-02-22 10:03:13 +00:00
|
|
|
let list: Vec<Value> = data
|
|
|
|
.as_array()
|
|
|
|
.unwrap()
|
|
|
|
.iter()
|
|
|
|
.filter(|item| item["keep"].as_bool() == Some(false))
|
|
|
|
.cloned()
|
|
|
|
.collect();
|
2020-06-03 08:11:37 +00:00
|
|
|
data = list.into();
|
|
|
|
}
|
|
|
|
|
2020-12-18 11:26:07 +00:00
|
|
|
format_and_print_result_full(&mut data, return_type, &output_format, &options);
|
2019-03-13 08:47:12 +00:00
|
|
|
|
2019-07-31 08:15:16 +00:00
|
|
|
Ok(Value::Null)
|
2019-02-27 15:53:17 +00:00
|
|
|
}
|
|
|
|
|
2019-12-16 12:34:49 +00:00
|
|
|
#[api(
|
|
|
|
input: {
|
|
|
|
properties: {
|
|
|
|
repository: {
|
|
|
|
schema: REPO_URL_SCHEMA,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
"output-format": {
|
|
|
|
schema: OUTPUT_FORMAT,
|
|
|
|
optional: true,
|
|
|
|
},
|
|
|
|
}
|
2020-10-28 21:59:39 +00:00
|
|
|
},
|
|
|
|
returns: {
|
|
|
|
type: StorageStatus,
|
|
|
|
},
|
2019-12-16 12:34:49 +00:00
|
|
|
)]
|
|
|
|
/// Get repository status.
|
|
|
|
async fn status(param: Value) -> Result<Value, Error> {
|
2019-07-16 11:35:25 +00:00
|
|
|
let repo = extract_repository_from_value(¶m)?;
|
|
|
|
|
2020-02-27 11:41:15 +00:00
|
|
|
let output_format = get_output_format(¶m);
|
2019-07-16 11:35:25 +00:00
|
|
|
|
2020-11-10 10:54:50 +00:00
|
|
|
let client = connect(&repo)?;
|
2019-07-16 11:35:25 +00:00
|
|
|
|
|
|
|
let path = format!("api2/json/admin/datastore/{}/status", repo.store());
|
|
|
|
|
2020-01-23 11:42:40 +00:00
|
|
|
let mut result = client.get(&path, None).await?;
|
2020-10-29 10:51:26 +00:00
|
|
|
let mut data = result["data"].take();
|
2019-07-16 11:35:25 +00:00
|
|
|
|
|
|
|
record_repository(&repo);
|
|
|
|
|
2020-02-26 12:29:00 +00:00
|
|
|
let render_total_percentage = |v: &Value, record: &Value| -> Result<String, Error> {
|
|
|
|
let v = v.as_u64().unwrap();
|
|
|
|
let total = record["total"].as_u64().unwrap();
|
2022-02-22 10:03:13 +00:00
|
|
|
let roundup = total / 200;
|
|
|
|
let per = ((v + roundup) * 100) / total;
|
2020-02-27 10:38:09 +00:00
|
|
|
let info = format!(" ({} %)", per);
|
|
|
|
Ok(format!("{} {:>8}", v, info))
|
2020-02-26 12:29:00 +00:00
|
|
|
};
|
2020-01-23 11:42:40 +00:00
|
|
|
|
2020-02-27 11:41:15 +00:00
|
|
|
let options = default_table_format_options()
|
2020-02-27 10:31:05 +00:00
|
|
|
.noheader(true)
|
2020-02-27 10:38:09 +00:00
|
|
|
.column(ColumnConfig::new("total").renderer(render_total_percentage))
|
2020-02-26 12:29:00 +00:00
|
|
|
.column(ColumnConfig::new("used").renderer(render_total_percentage))
|
|
|
|
.column(ColumnConfig::new("avail").renderer(render_total_percentage));
|
2019-07-16 11:35:25 +00:00
|
|
|
|
2020-12-18 11:26:07 +00:00
|
|
|
let return_type = &API_METHOD_STATUS.returns;
|
2020-02-26 12:29:00 +00:00
|
|
|
|
2020-12-18 11:26:07 +00:00
|
|
|
format_and_print_result_full(&mut data, return_type, &output_format, &options);
|
2019-07-16 11:35:25 +00:00
|
|
|
|
|
|
|
Ok(Value::Null)
|
|
|
|
}
|
|
|
|
|
2020-03-23 14:03:18 +00:00
|
|
|
/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
|
|
|
|
/// async use!
|
|
|
|
///
|
|
|
|
/// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
|
|
|
|
/// so that we can properly access it from multiple threads simultaneously while not issuing
|
|
|
|
/// duplicate simultaneous reads over http.
|
2020-07-02 15:49:08 +00:00
|
|
|
pub struct BufferedDynamicReadAt {
|
2020-03-23 14:03:18 +00:00
|
|
|
inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl BufferedDynamicReadAt {
|
|
|
|
fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
|
|
|
|
Self {
|
|
|
|
inner: Mutex::new(inner),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-24 09:57:12 +00:00
|
|
|
impl ReadAt for BufferedDynamicReadAt {
|
|
|
|
fn start_read_at<'a>(
|
|
|
|
self: Pin<&'a Self>,
|
2020-03-23 14:03:18 +00:00
|
|
|
_cx: &mut Context,
|
2020-06-24 09:57:12 +00:00
|
|
|
buf: &'a mut [u8],
|
2020-03-23 14:03:18 +00:00
|
|
|
offset: u64,
|
2020-06-24 09:57:12 +00:00
|
|
|
) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
|
|
|
|
MaybeReady::Ready(tokio::task::block_in_place(move || {
|
2020-03-23 14:03:18 +00:00
|
|
|
let mut reader = self.inner.lock().unwrap();
|
|
|
|
reader.seek(SeekFrom::Start(offset))?;
|
2022-02-08 13:57:16 +00:00
|
|
|
reader.read(buf)
|
2020-06-24 09:57:12 +00:00
|
|
|
}))
|
|
|
|
}
|
|
|
|
|
|
|
|
fn poll_complete<'a>(
|
|
|
|
self: Pin<&'a Self>,
|
|
|
|
_op: ReadAtOperation<'a>,
|
|
|
|
) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
|
2021-07-21 11:28:55 +00:00
|
|
|
panic!("BufferedDynamicReadAt::start_read_at returned Pending");
|
2020-03-23 14:03:18 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-06-17 08:33:24 +00:00
|
|
|
fn main() {
|
Set MMAP_THRESHOLD to a fixed value (128K)
glibc's malloc has a misguided heuristic to detect transient allocations that
will just result in allocation sizes below 32 MiB never using mmap.
That it turn means that those relatively big allocations are on the heap where
cleanup and returning memory to the OS is harder to do and easier to be blocked
by long living, small allocations at the top (end) of the heap.
Observing the malloc size distribution in a file-level backup run:
@size:
[0] 14 | |
[1] 25214 |@@@@@ |
[2, 4) 9090 |@ |
[4, 8) 12987 |@@ |
[8, 16) 93453 |@@@@@@@@@@@@@@@@@@@@ |
[16, 32) 30255 |@@@@@@ |
[32, 64) 237445 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
[64, 128) 32692 |@@@@@@@ |
[128, 256) 22296 |@@@@ |
[256, 512) 16177 |@@@ |
[512, 1K) 5139 |@ |
[1K, 2K) 3352 | |
[2K, 4K) 214 | |
[4K, 8K) 1568 | |
[8K, 16K) 95 | |
[16K, 32K) 3457 | |
[32K, 64K) 3175 | |
[64K, 128K) 161 | |
[128K, 256K) 453 | |
[256K, 512K) 93 | |
[512K, 1M) 74 | |
[1M, 2M) 774 | |
[2M, 4M) 319 | |
[4M, 8M) 700 | |
[8M, 16M) 93 | |
[16M, 32M) 18 | |
We see that all allocations will be on the heap, and that while most
allocations are small, the relatively few big ones will still make up most of
the RSS and if blocked from being released back to the OS result in much higher
peak and average usage for the program than actually required.
Avoiding the "dynamic" mmap-threshold increasement algorithm and fixing it at
the original default of 128 KiB reduces RSS size by factor 10-20 when running
backups. As with memory mappings other mappings or the heap can never block
freeing the memory fully back to the OS.
But, the drawback of using mmap is more wasted space for unaligned or small
allocation sizes, and the fact that the kernel allegedly zeros out the data
before giving it to user space. The former doesn't really matter for us when
using it only for allocations bigger than 128 KiB, and the latter is a
trade-off, using 10 to 20 times less memory brings its own performance
improvement possibilities for the whole system after all ;-)
Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
[ Thomas: added to comment & commit message + extra-empty-line fixes ]
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
2022-01-26 06:10:59 +00:00
|
|
|
pbs_tools::setup_libc_malloc_opts();
|
2019-02-13 11:30:52 +00:00
|
|
|
|
2019-11-21 08:36:41 +00:00
|
|
|
let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
|
2019-11-24 10:00:53 +00:00
|
|
|
.arg_param(&["backupspec"])
|
2019-03-13 08:47:12 +00:00
|
|
|
.completion_cb("repository", complete_repository)
|
2019-03-13 11:26:01 +00:00
|
|
|
.completion_cb("backupspec", complete_backup_source)
|
2021-10-13 10:55:51 +00:00
|
|
|
.completion_cb("keyfile", complete_file_name)
|
|
|
|
.completion_cb("master-pubkey-file", complete_file_name)
|
2019-03-13 11:26:01 +00:00
|
|
|
.completion_cb("chunk-size", complete_chunk_size);
|
2018-12-15 10:24:39 +00:00
|
|
|
|
2020-07-02 12:00:32 +00:00
|
|
|
let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
|
|
|
|
.completion_cb("repository", complete_repository)
|
2021-10-13 10:55:51 +00:00
|
|
|
.completion_cb("keyfile", complete_file_name);
|
2020-07-02 12:00:32 +00:00
|
|
|
|
2019-11-21 08:36:41 +00:00
|
|
|
let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
|
2019-03-13 08:47:12 +00:00
|
|
|
.completion_cb("repository", complete_repository);
|
2019-01-21 17:58:14 +00:00
|
|
|
|
2019-11-21 08:36:41 +00:00
|
|
|
let garbage_collect_cmd_def = CliCommand::new(&API_METHOD_START_GARBAGE_COLLECTION)
|
2019-03-13 08:47:12 +00:00
|
|
|
.completion_cb("repository", complete_repository);
|
2019-02-20 13:10:45 +00:00
|
|
|
|
2019-11-21 08:36:41 +00:00
|
|
|
let restore_cmd_def = CliCommand::new(&API_METHOD_RESTORE)
|
2019-11-24 10:00:53 +00:00
|
|
|
.arg_param(&["snapshot", "archive-name", "target"])
|
2019-03-13 12:31:29 +00:00
|
|
|
.completion_cb("repository", complete_repository)
|
2019-03-14 07:09:35 +00:00
|
|
|
.completion_cb("snapshot", complete_group_or_snapshot)
|
|
|
|
.completion_cb("archive-name", complete_archive_name)
|
2021-10-13 10:55:51 +00:00
|
|
|
.completion_cb("target", complete_file_name);
|
2019-03-06 09:50:46 +00:00
|
|
|
|
2019-11-21 08:36:41 +00:00
|
|
|
let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
|
2019-11-24 10:00:53 +00:00
|
|
|
.arg_param(&["group"])
|
2019-07-27 06:49:14 +00:00
|
|
|
.completion_cb("group", complete_backup_group)
|
2019-03-13 08:47:12 +00:00
|
|
|
.completion_cb("repository", complete_repository);
|
2019-03-06 09:50:46 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let status_cmd_def =
|
|
|
|
CliCommand::new(&API_METHOD_STATUS).completion_cb("repository", complete_repository);
|
2019-07-16 11:35:25 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let login_cmd_def =
|
|
|
|
CliCommand::new(&API_METHOD_API_LOGIN).completion_cb("repository", complete_repository);
|
2019-08-10 07:12:17 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let logout_cmd_def =
|
|
|
|
CliCommand::new(&API_METHOD_API_LOGOUT).completion_cb("repository", complete_repository);
|
2019-09-20 06:12:31 +00:00
|
|
|
|
2022-02-22 10:03:13 +00:00
|
|
|
let version_cmd_def =
|
|
|
|
CliCommand::new(&API_METHOD_API_VERSION).completion_cb("repository", complete_repository);
|
2020-07-10 07:34:07 +00:00
|
|
|
|
2020-10-13 08:58:41 +00:00
|
|
|
let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
|
|
|
|
.arg_param(&["group", "new-owner"])
|
|
|
|
.completion_cb("group", complete_backup_group)
|
2022-02-22 10:03:13 +00:00
|
|
|
.completion_cb("new-owner", complete_auth_id)
|
2020-10-13 08:58:41 +00:00
|
|
|
.completion_cb("repository", complete_repository);
|
|
|
|
|
2019-01-21 17:58:14 +00:00
|
|
|
let cmd_def = CliCommandMap::new()
|
2019-12-09 16:40:34 +00:00
|
|
|
.insert("backup", backup_cmd_def)
|
|
|
|
.insert("garbage-collect", garbage_collect_cmd_def)
|
|
|
|
.insert("list", list_cmd_def)
|
|
|
|
.insert("login", login_cmd_def)
|
|
|
|
.insert("logout", logout_cmd_def)
|
|
|
|
.insert("prune", prune_cmd_def)
|
|
|
|
.insert("restore", restore_cmd_def)
|
2020-11-24 09:09:29 +00:00
|
|
|
.insert("snapshot", snapshot_mgtm_cli())
|
2019-12-09 16:40:34 +00:00
|
|
|
.insert("status", status_cmd_def)
|
2020-07-06 09:39:24 +00:00
|
|
|
.insert("key", key::cli())
|
2020-07-02 15:49:08 +00:00
|
|
|
.insert("mount", mount_cmd_def())
|
client: implement map/unmap commands for .img backups
Allows mapping fixed-index .img files (usually from VM backups) to be
mapped to a local loopback device.
The architecture uses a FUSE-backed temp file mapped to a loopdev:
/dev/loopX -> FUSE /run/pbs-loopdev/xxx -> backup client -> PBS
Since unmapping requires some cleanup (unmap the loopdev, stop FUSE,
remove the temp files) a special 'unmap' command is added, which uses a
PID file to send SIGINT to the backup-client instance started with
'map', which will handle the cleanup itself.
The polling with select! in mount.rs needs to be split in two, since we
have a chicken and egg problem between running FUSE and setting up the
loop device - so we need to do them concurrently, until the loopdev is
assigned, at which point we can report success and daemonize, and then
continue polling the FUSE loop future.
A loopdev module is added to tools containing all required functions for
mapping a loop device to the FUSE file, with the ioctls moved into an
inline module to avoid exposing them directly.
The client code is placed in the 'mount' module, which, while
admittedly a loose fit, allows reuse of the daemonizing code.
Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
2020-10-05 08:57:58 +00:00
|
|
|
.insert("map", map_cmd_def())
|
|
|
|
.insert("unmap", unmap_cmd_def())
|
2019-12-10 12:43:53 +00:00
|
|
|
.insert("catalog", catalog_mgmt_cli())
|
2020-07-02 12:00:32 +00:00
|
|
|
.insert("task", task_mgmt_cli())
|
2020-07-10 07:34:07 +00:00
|
|
|
.insert("version", version_cmd_def)
|
2020-10-13 08:58:41 +00:00
|
|
|
.insert("benchmark", benchmark_cmd_def)
|
2020-11-24 12:01:06 +00:00
|
|
|
.insert("change-owner", change_owner_cmd_def)
|
2020-11-25 05:47:18 +00:00
|
|
|
.alias(&["files"], &["snapshot", "files"])
|
2020-11-25 05:51:23 +00:00
|
|
|
.alias(&["forget"], &["snapshot", "forget"])
|
2020-11-25 06:06:55 +00:00
|
|
|
.alias(&["upload-log"], &["snapshot", "upload-log"])
|
2022-02-22 10:03:13 +00:00
|
|
|
.alias(&["snapshots"], &["snapshot", "list"]);
|
2019-12-09 16:40:34 +00:00
|
|
|
|
2020-05-06 05:51:05 +00:00
|
|
|
let rpcenv = CliEnvironment::new();
|
2022-02-22 10:03:13 +00:00
|
|
|
run_cli_command(
|
|
|
|
cmd_def,
|
|
|
|
rpcenv,
|
|
|
|
Some(|future| proxmox_async::runtime::main(future)),
|
|
|
|
);
|
2018-12-14 07:28:56 +00:00
|
|
|
}
|