2020-01-10 10:09:55 +00:00
|
|
|
//! Sync datastore from remote server
|
2020-05-29 08:53:36 +00:00
|
|
|
use std::sync::{Arc};
|
2020-01-10 10:09:55 +00:00
|
|
|
|
2020-05-22 06:04:20 +00:00
|
|
|
use anyhow::{format_err, Error};
|
2020-08-13 08:29:18 +00:00
|
|
|
use futures::{select, future::FutureExt};
|
2020-01-08 12:53:19 +00:00
|
|
|
|
|
|
|
use proxmox::api::api;
|
2020-04-17 13:27:04 +00:00
|
|
|
use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
|
2020-01-08 12:53:19 +00:00
|
|
|
|
|
|
|
use crate::server::{WorkerTask};
|
2020-05-22 06:04:20 +00:00
|
|
|
use crate::backup::DataStore;
|
|
|
|
use crate::client::{HttpClient, HttpClientOptions, BackupRepository, pull::pull_store};
|
2020-01-08 12:53:19 +00:00
|
|
|
use crate::api2::types::*;
|
2020-05-22 06:04:20 +00:00
|
|
|
use crate::config::{
|
|
|
|
remote,
|
2020-08-13 08:29:17 +00:00
|
|
|
sync::SyncJobConfig,
|
2020-08-13 08:29:18 +00:00
|
|
|
jobstate::Job,
|
2020-05-22 06:04:20 +00:00
|
|
|
acl::{PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ},
|
|
|
|
cached_user_info::CachedUserInfo,
|
|
|
|
};
|
2020-01-08 12:53:19 +00:00
|
|
|
|
2020-05-29 08:53:36 +00:00
|
|
|
|
|
|
|
pub fn check_pull_privs(
|
2020-08-06 13:46:01 +00:00
|
|
|
userid: &Userid,
|
2020-05-29 08:53:36 +00:00
|
|
|
store: &str,
|
|
|
|
remote: &str,
|
|
|
|
remote_store: &str,
|
|
|
|
delete: bool,
|
|
|
|
) -> Result<(), Error> {
|
|
|
|
|
|
|
|
let user_info = CachedUserInfo::new()?;
|
|
|
|
|
2020-08-06 13:46:01 +00:00
|
|
|
user_info.check_privs(userid, &["datastore", store], PRIV_DATASTORE_BACKUP, false)?;
|
|
|
|
user_info.check_privs(userid, &["remote", remote, remote_store], PRIV_REMOTE_READ, false)?;
|
2020-05-29 08:53:36 +00:00
|
|
|
|
|
|
|
if delete {
|
2020-08-06 13:46:01 +00:00
|
|
|
user_info.check_privs(userid, &["datastore", store], PRIV_DATASTORE_PRUNE, false)?;
|
2020-05-29 08:53:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
pub async fn get_pull_parameters(
|
|
|
|
store: &str,
|
|
|
|
remote: &str,
|
|
|
|
remote_store: &str,
|
|
|
|
) -> Result<(HttpClient, BackupRepository, Arc<DataStore>), Error> {
|
|
|
|
|
|
|
|
let tgt_store = DataStore::lookup_datastore(store)?;
|
|
|
|
|
|
|
|
let (remote_config, _digest) = remote::config()?;
|
|
|
|
let remote: remote::Remote = remote_config.lookup("remote", remote)?;
|
|
|
|
|
|
|
|
let options = HttpClientOptions::new()
|
|
|
|
.password(Some(remote.password.clone()))
|
|
|
|
.fingerprint(remote.fingerprint.clone());
|
|
|
|
|
2020-09-30 11:23:39 +00:00
|
|
|
let src_repo = BackupRepository::new(Some(remote.userid.clone()), Some(remote.host.clone()), remote.port, remote_store.to_string());
|
|
|
|
|
|
|
|
let client = HttpClient::new(&src_repo.host(), src_repo.port(), &src_repo.user(), options)?;
|
2020-05-29 08:53:36 +00:00
|
|
|
let _auth_info = client.login() // make sure we can auth
|
|
|
|
.await
|
|
|
|
.map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?;
|
|
|
|
|
|
|
|
|
|
|
|
Ok((client, src_repo, tgt_store))
|
|
|
|
}
|
|
|
|
|
2020-08-13 08:29:17 +00:00
|
|
|
pub fn do_sync_job(
|
2020-08-13 12:30:18 +00:00
|
|
|
mut job: Job,
|
2020-08-13 08:29:17 +00:00
|
|
|
sync_job: SyncJobConfig,
|
|
|
|
userid: &Userid,
|
2020-08-13 08:29:18 +00:00
|
|
|
schedule: Option<String>,
|
2020-08-13 08:29:17 +00:00
|
|
|
) -> Result<String, Error> {
|
|
|
|
|
2020-08-13 12:30:18 +00:00
|
|
|
let job_id = job.jobname().to_string();
|
|
|
|
let worker_type = job.jobtype().to_string();
|
2020-08-13 08:29:17 +00:00
|
|
|
|
2020-08-13 08:29:18 +00:00
|
|
|
let upid_str = WorkerTask::spawn(
|
2020-08-13 12:30:18 +00:00
|
|
|
&worker_type,
|
|
|
|
Some(job.jobname().to_string()),
|
2020-08-13 08:29:18 +00:00
|
|
|
userid.clone(),
|
|
|
|
false,
|
|
|
|
move |worker| async move {
|
2020-08-13 08:29:17 +00:00
|
|
|
|
2020-08-13 08:29:18 +00:00
|
|
|
job.start(&worker.upid().to_string())?;
|
2020-08-13 08:29:17 +00:00
|
|
|
|
2020-08-13 08:29:18 +00:00
|
|
|
let worker2 = worker.clone();
|
2020-08-13 08:29:17 +00:00
|
|
|
|
2020-08-13 08:29:18 +00:00
|
|
|
let worker_future = async move {
|
2020-08-13 08:29:17 +00:00
|
|
|
|
2020-08-13 08:29:18 +00:00
|
|
|
let delete = sync_job.remove_vanished.unwrap_or(true);
|
|
|
|
let (client, src_repo, tgt_store) = get_pull_parameters(&sync_job.store, &sync_job.remote, &sync_job.remote_store).await?;
|
|
|
|
|
|
|
|
worker.log(format!("Starting datastore sync job '{}'", job_id));
|
|
|
|
if let Some(event_str) = schedule {
|
|
|
|
worker.log(format!("task triggered by schedule '{}'", event_str));
|
|
|
|
}
|
|
|
|
worker.log(format!("Sync datastore '{}' from '{}/{}'",
|
|
|
|
sync_job.store, sync_job.remote, sync_job.remote_store));
|
|
|
|
|
|
|
|
crate::client::pull::pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, Userid::backup_userid().clone()).await?;
|
|
|
|
|
|
|
|
worker.log(format!("sync job '{}' end", &job_id));
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
};
|
|
|
|
|
|
|
|
let mut abort_future = worker2.abort_future().map(|_| Err(format_err!("sync aborted")));
|
|
|
|
|
|
|
|
let res = select!{
|
|
|
|
worker = worker_future.fuse() => worker,
|
|
|
|
abort = abort_future => abort,
|
|
|
|
};
|
|
|
|
|
|
|
|
let status = worker2.create_state(&res);
|
|
|
|
|
|
|
|
match job.finish(status) {
|
|
|
|
Ok(_) => {},
|
|
|
|
Err(err) => {
|
|
|
|
eprintln!("could not finish job state: {}", err);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
res
|
|
|
|
})?;
|
2020-08-13 08:29:17 +00:00
|
|
|
|
|
|
|
Ok(upid_str)
|
|
|
|
}
|
|
|
|
|
2020-01-08 12:53:19 +00:00
|
|
|
#[api(
|
|
|
|
input: {
|
|
|
|
properties: {
|
|
|
|
store: {
|
|
|
|
schema: DATASTORE_SCHEMA,
|
|
|
|
},
|
2020-01-16 12:54:29 +00:00
|
|
|
remote: {
|
|
|
|
schema: REMOTE_ID_SCHEMA,
|
2020-01-08 12:53:19 +00:00
|
|
|
},
|
|
|
|
"remote-store": {
|
|
|
|
schema: DATASTORE_SCHEMA,
|
|
|
|
},
|
2020-05-21 06:31:16 +00:00
|
|
|
"remove-vanished": {
|
|
|
|
schema: REMOVE_VANISHED_BACKUPS_SCHEMA,
|
2020-01-17 10:24:55 +00:00
|
|
|
optional: true,
|
|
|
|
},
|
2020-01-08 12:53:19 +00:00
|
|
|
},
|
|
|
|
},
|
2020-04-17 13:27:04 +00:00
|
|
|
access: {
|
2020-04-18 06:23:04 +00:00
|
|
|
// Note: used parameters are no uri parameters, so we need to test inside function body
|
2020-04-28 08:11:15 +00:00
|
|
|
description: r###"The user needs Datastore.Backup privilege on '/datastore/{store}',
|
2020-04-29 05:03:44 +00:00
|
|
|
and needs to own the backup group. Remote.Read is required on '/remote/{remote}/{remote-store}'.
|
2020-04-29 05:11:03 +00:00
|
|
|
The delete flag additionally requires the Datastore.Prune privilege on '/datastore/{store}'.
|
2020-04-28 08:11:15 +00:00
|
|
|
"###,
|
2020-04-18 06:23:04 +00:00
|
|
|
permission: &Permission::Anybody,
|
2020-04-17 13:27:04 +00:00
|
|
|
},
|
2020-01-08 12:53:19 +00:00
|
|
|
)]
|
2020-01-10 10:09:55 +00:00
|
|
|
/// Sync store from other repository
|
|
|
|
async fn pull (
|
2020-01-08 12:53:19 +00:00
|
|
|
store: String,
|
2020-01-16 12:54:29 +00:00
|
|
|
remote: String,
|
2020-01-08 12:53:19 +00:00
|
|
|
remote_store: String,
|
2020-05-21 06:31:16 +00:00
|
|
|
remove_vanished: Option<bool>,
|
2020-01-08 12:53:19 +00:00
|
|
|
_info: &ApiMethod,
|
|
|
|
rpcenv: &mut dyn RpcEnvironment,
|
|
|
|
) -> Result<String, Error> {
|
|
|
|
|
2020-08-06 13:46:01 +00:00
|
|
|
let userid: Userid = rpcenv.get_user().unwrap().parse()?;
|
2020-05-21 06:31:16 +00:00
|
|
|
let delete = remove_vanished.unwrap_or(true);
|
2020-01-17 10:24:55 +00:00
|
|
|
|
2020-08-06 13:46:01 +00:00
|
|
|
check_pull_privs(&userid, &store, &remote, &remote_store, delete)?;
|
2020-01-08 12:53:19 +00:00
|
|
|
|
2020-05-29 08:53:36 +00:00
|
|
|
let (client, src_repo, tgt_store) = get_pull_parameters(&store, &remote, &remote_store).await?;
|
2020-01-08 12:53:19 +00:00
|
|
|
|
|
|
|
// fixme: set to_stdout to false?
|
2020-08-06 13:46:01 +00:00
|
|
|
let upid_str = WorkerTask::spawn("sync", Some(store.clone()), userid.clone(), true, move |worker| async move {
|
2020-01-08 12:53:19 +00:00
|
|
|
|
|
|
|
worker.log(format!("sync datastore '{}' start", store));
|
|
|
|
|
2020-09-15 09:15:01 +00:00
|
|
|
let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, userid);
|
|
|
|
let future = select!{
|
|
|
|
success = pull_future.fuse() => success,
|
|
|
|
abort = worker.abort_future().map(|_| Err(format_err!("pull aborted"))) => abort,
|
|
|
|
};
|
|
|
|
|
|
|
|
let _ = future?;
|
2020-01-08 12:53:19 +00:00
|
|
|
|
|
|
|
worker.log(format!("sync datastore '{}' end", store));
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
})?;
|
|
|
|
|
|
|
|
Ok(upid_str)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub const ROUTER: Router = Router::new()
|
2020-01-10 10:09:55 +00:00
|
|
|
.post(&API_METHOD_PULL);
|