move more tools for the client into subcrates

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>

Cargo.toml

@@ -44,7 +44,6 @@ endian_trait = { version = "0.6", features = ["arrays"] }
 env_logger = "0.7"
 flate2 = "1.0"
 anyhow = "1.0"
-foreign-types = "0.3"
 thiserror = "1.0"
 futures = "0.3"
 h2 = { version = "0.3", features = [ "stream" ] }

pbs-tools/Cargo.toml

@@ -9,6 +9,9 @@ description = "common tools used throughout pbs"
 [dependencies]
 anyhow = "1.0"
 base64 = "0.12"
+foreign-types = "0.3"
+futures = "0.3"
+lazy_static = "1.4"
 libc = "0.2"
 nix = "0.19.1"
 nom = "5.1"
@@ -17,6 +20,10 @@ percent-encoding = "2.1"
 regex = "1.2"
 serde = "1.0"
 serde_json = "1.0"
+# rt-multi-thread is required for block_in_place
+tokio = { version = "1.6", features = [ "rt", "rt-multi-thread", "sync" ] }
 url = "2.1"
 
 proxmox = { version = "0.11.5", default-features = false, features = [] }
+
+pbs-buildcfg = { path = "../pbs-buildcfg" }

pbs-tools/src/auth.rs

@@ -0,0 +1,26 @@
+//! Helpers for authentication used by both client and server.
+
+use anyhow::Error;
+use lazy_static::lazy_static;
+use openssl::pkey::{PKey, Private};
+use openssl::rsa::Rsa;
+
+use proxmox::tools::fs::file_get_contents;
+
+use pbs_buildcfg::configdir;
+
+fn load_private_auth_key() -> Result<PKey<Private>, Error> {
+    let pem = file_get_contents(configdir!("/authkey.key"))?;
+    let rsa = Rsa::private_key_from_pem(&pem)?;
+    let key = PKey::from_rsa(rsa)?;
+
+    Ok(key)
+}
+
+pub fn private_auth_key() -> &'static PKey<Private> {
+    lazy_static! {
+        static ref KEY: PKey<Private> = load_private_auth_key().unwrap();
+    }
+
+    &KEY
+}

pbs-tools/src/lib.rs

@@ -1,12 +1,18 @@
+pub mod auth;
 pub mod borrow;
+pub mod broadcast_future;
+pub mod cert;
 pub mod format;
 pub mod fs;
 pub mod json;
 pub mod nom;
+pub mod percent_encoding;
 pub mod process_locker;
 pub mod sha;
 pub mod str;
+pub mod sync;
 pub mod ticket;
+pub mod tokio;
 
 mod command;
 pub use command::{command_output, command_output_as_string, run_command};

pbs-tools/src/percent_encoding.rs

@@ -0,0 +1,22 @@
+use percent_encoding::{utf8_percent_encode, AsciiSet};
+
+/// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
+pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
+    // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
+    .add(0x20)
+    .add(0x7f)
+    // the DEFAULT_ENCODE_SET added:
+    .add(b' ')
+    .add(b'"')
+    .add(b'#')
+    .add(b'<')
+    .add(b'>')
+    .add(b'`')
+    .add(b'?')
+    .add(b'{')
+    .add(b'}');
+
+/// percent encode a url component
+pub fn percent_encode_component(comp: &str) -> String {
+    utf8_percent_encode(comp, percent_encoding::NON_ALPHANUMERIC).to_string()
+}

pbs-tools/src/sync/mod.rs

@@ -0,0 +1,2 @@
+mod std_channel_writer;
+pub use std_channel_writer::StdChannelWriter;

pbs-tools/src/tokio/mod.rs

@@ -0,0 +1,2 @@
+pub mod tokio_writer_adapter;
+pub use tokio_writer_adapter::TokioWriterAdapter;

src/api2/access/mod.rs

@@ -11,6 +11,7 @@ use proxmox::api::{api, Permission, RpcEnvironment};
 use proxmox::{http_err, list_subdirs_api_method};
 use proxmox::{identity, sortable};
 
+use pbs_tools::auth::private_auth_key;
 use pbs_tools::ticket::{self, Empty, Ticket};
 
 use crate::api2::types::*;

src/api2/access/openid.rs

@@ -14,6 +14,7 @@ use proxmox::tools::fs::open_file_locked;
 use proxmox_openid::{OpenIdAuthenticator,  OpenIdConfig};
 
 use pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M;
+use pbs_tools::auth::private_auth_key;
 use pbs_tools::ticket::Ticket;
 
 use crate::server::ticket::ApiTicket;

src/api2/node/certificates.rs

@@ -12,6 +12,7 @@ use proxmox::api::{api, Permission, Router, RpcEnvironment};
 use proxmox::list_subdirs_api_method;
 
 use pbs_buildcfg::configdir;
+use pbs_tools::cert;
 
 use crate::acme::AcmeClient;
 use crate::api2::types::Authid;
@@ -20,7 +21,6 @@ use crate::api2::types::AcmeDomain;
 use crate::config::acl::PRIV_SYS_MODIFY;
 use crate::config::node::NodeConfig;
 use crate::server::WorkerTask;
-use crate::tools::cert;
 
 pub const ROUTER: Router = Router::new()
     .get(&list_subdirs_api_method!(SUBDIRS))

src/api2/node/mod.rs

@@ -20,6 +20,7 @@ use proxmox::list_subdirs_api_method;
 use proxmox_http::websocket::WebSocket;
 use proxmox::{identity, sortable};
 
+use pbs_tools::auth::private_auth_key;
 use pbs_tools::ticket::{self, Empty, Ticket};
 
 use crate::api2::types::*;
@@ -121,7 +122,7 @@ async fn termproxy(
 
     let ticket = Ticket::new(ticket::TERM_PREFIX, &Empty)?
         .sign(
-            crate::auth_helpers::private_auth_key(),
+            private_auth_key(),
             Some(&tools::ticket::term_aad(&userid, &path, port)),
         )?;
 

src/api2/node/status.rs

@@ -8,9 +8,10 @@ use proxmox::sys::linux::procfs;
 
 use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
 
+use pbs_tools::cert::CertInfo;
+
 use crate::api2::types::*;
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_POWER_MANAGEMENT};
-use crate::tools::cert::CertInfo;
 
 impl std::convert::From<procfs::ProcFsCPUInfo> for NodeCpuInformation {
     fn from(info: procfs::ProcFsCPUInfo) -> Self {

src/auth_helpers.rs

@@ -1,18 +1,16 @@
+use std::path::PathBuf;
+
 use anyhow::{bail, format_err, Error};
 use lazy_static::lazy_static;
-
+use openssl::pkey::{PKey, Public};
-use openssl::rsa::{Rsa};
+use openssl::rsa::Rsa;
-use openssl::pkey::{PKey, Public, Private};
 use openssl::sha;
 
-use std::path::PathBuf;
-
 use proxmox::tools::fs::{file_get_contents, replace_file, CreateOptions};
 use proxmox::try_block;
 
 use pbs_buildcfg::configdir;
-
+use pbs_api_types::Userid;
-use crate::api2::types::Userid;
 
 fn compute_csrf_secret_digest(
     timestamp: i64,
@@ -155,24 +153,6 @@ pub fn csrf_secret() -> &'static [u8] {
     &SECRET
 }
 
-fn load_private_auth_key() -> Result<PKey<Private>, Error> {
-
-    let pem = file_get_contents(configdir!("/authkey.key"))?;
-    let rsa = Rsa::private_key_from_pem(&pem)?;
-    let key = PKey::from_rsa(rsa)?;
-
-    Ok(key)
-}
-
-pub fn private_auth_key() -> &'static PKey<Private> {
-
-    lazy_static! {
-        static ref KEY: PKey<Private> = load_private_auth_key().unwrap();
-    }
-
-    &KEY
-}
-
 fn load_public_auth_key() -> Result<PKey<Public>, Error> {
 
     let pem = file_get_contents(configdir!("/authkey.pub"))?;

src/bin/proxmox-backup-api.rs

@@ -4,8 +4,8 @@ use futures::*;
 use proxmox::try_block;
 use proxmox::api::RpcEnvironmentType;
 
-//use proxmox_backup::tools;
+use pbs_tools::auth::private_auth_key;
-//use proxmox_backup::api_schema::config::*;
+
 use proxmox_backup::server::{
     self,
     auth::default_api_auth,

src/bin/proxmox-backup-client.rs

@@ -28,12 +28,9 @@ use proxmox::{
 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
 
 use pbs_datastore::catalog::BackupCatalogWriter;
+use pbs_tools::sync::StdChannelWriter;
+use pbs_tools::tokio::TokioWriterAdapter;
 
-use proxmox_backup::tools::{
-    self,
-    StdChannelWriter,
-    TokioWriterAdapter,
-};
 use proxmox_backup::api2::types::*;
 use proxmox_backup::api2::version;
 use proxmox_backup::client::*;
@@ -64,6 +61,7 @@ use proxmox_backup::backup::{
     Shell,
     PruneOptions,
 };
+use proxmox_backup::tools;
 
 mod proxmox_backup_client;
 use proxmox_backup_client::*;

src/bin/proxmox-backup-manager.rs

@@ -6,6 +6,8 @@ use serde_json::{json, Value};
 
 use proxmox::api::{api, cli::*, RpcEnvironment};
 
+use pbs_tools::percent_encoding::percent_encode_component;
+
 use proxmox_backup::tools;
 use proxmox_backup::config;
 use proxmox_backup::api2::{self, types::* };
@@ -188,7 +190,7 @@ async fn task_stop(param: Value) -> Result<Value, Error> {
 
     let mut client = connect_to_localhost()?;
 
-    let path = format!("api2/json/nodes/localhost/tasks/{}", tools::percent_encode_component(upid_str));
+    let path = format!("api2/json/nodes/localhost/tasks/{}", percent_encode_component(upid_str));
     let _ = client.delete(&path, None).await?;
 
     Ok(Value::Null)

src/bin/proxmox_backup_client/task.rs

@@ -3,6 +3,8 @@ use serde_json::{json, Value};
 
 use proxmox::api::{api, cli::*};
 
+use pbs_tools::percent_encoding::percent_encode_component;
+
 use proxmox_backup::tools;
 
 use proxmox_backup::client::*;
@@ -125,7 +127,7 @@ async fn task_stop(param: Value) -> Result<Value, Error> {
 
     let mut client = connect(&repo)?;
 
-    let path = format!("api2/json/nodes/localhost/tasks/{}", tools::percent_encode_component(upid_str));
+    let path = format!("api2/json/nodes/localhost/tasks/{}", percent_encode_component(upid_str));
     let _ = client.delete(&path, None).await?;
 
     Ok(Value::Null)

src/bin/proxmox_backup_manager/cert.rs

@@ -2,9 +2,10 @@ use anyhow::{bail, Error};
 
 use proxmox::api::{api, cli::*};
 
+use pbs_tools::cert::CertInfo;
+
 use proxmox_backup::config;
 use proxmox_backup::auth_helpers::*;
-use proxmox_backup::tools::cert::CertInfo;
 
 #[api]
 /// Display node certificate information.

src/client/backup_writer.rs

@@ -14,10 +14,15 @@ use tokio_stream::wrappers::ReceiverStream;
 
 use proxmox::tools::digest_to_hex;
 
+use pbs_datastore::{CATALOG_NAME, CryptConfig};
+use pbs_datastore::data_blob::{ChunkInfo, DataBlob, DataChunkBuilder};
+use pbs_datastore::dynamic_index::DynamicIndexReader;
+use pbs_datastore::fixed_index::FixedIndexReader;
+use pbs_datastore::index::IndexFile;
+use pbs_datastore::manifest::{ArchiveType, BackupManifest, MANIFEST_BLOB_NAME};
 use pbs_tools::format::HumanByte;
 
 use super::merge_known_chunks::{MergeKnownChunks, MergedChunkInfo};
-use crate::backup::*;
 
 use super::{H2Client, HttpClient};
 
@@ -283,7 +288,7 @@ impl BackupWriter {
 
         if let Some(manifest) = options.previous_manifest {
             // try, but ignore errors
-            match archive_type(archive_name) {
+            match ArchiveType::from_path(archive_name) {
                 Ok(ArchiveType::FixedIndex) => {
                     let _ = self
                         .download_previous_fixed_index(

src/client/http_client.rs

@@ -24,15 +24,13 @@ use proxmox_http::client::HttpsConnector;
 use proxmox_http::uri::build_authority;
 
 use pbs_api_types::{Authid, Userid};
+use pbs_tools::broadcast_future::BroadcastFuture;
 use pbs_tools::json::json_object_to_query;
 use pbs_tools::ticket;
+use pbs_tools::percent_encoding::DEFAULT_ENCODE_SET;
 
 use super::pipe_to_stream::PipeToSendStream;
-use crate::tools::{
+use super::PROXMOX_BACKUP_TCP_KEEPALIVE_TIME;
-    BroadcastFuture,
-    DEFAULT_ENCODE_SET,
-    PROXMOX_BACKUP_TCP_KEEPALIVE_TIME,
-};
 
 /// Timeout used for several HTTP operations that are expected to finish quickly but may block in
 /// certain error conditions. Keep it generous, to avoid false-positive under high load.

src/client/mod.rs

@@ -7,11 +7,8 @@ use anyhow::Error;
 
 use pbs_api_types::{Authid, Userid};
 use pbs_tools::ticket::Ticket;
-
+use pbs_tools::cert::CertInfo;
-use crate::{
+use pbs_tools::auth::private_auth_key;
-    tools::cert::CertInfo,
-    auth_helpers::private_auth_key,
-};
 
 mod merge_known_chunks;
 pub mod pipe_to_stream;
@@ -43,6 +40,8 @@ pub use backup_repo::*;
 mod backup_specification;
 pub use backup_specification::*;
 
+pub const PROXMOX_BACKUP_TCP_KEEPALIVE_TIME: u32 = 120;
+
 /// Connect to localhost:8007 as root@pam
 ///
 /// This automatically creates a ticket if run as 'root' user.

src/client/pxar_backup_stream.rs

@@ -13,11 +13,8 @@ use nix::fcntl::OFlag;
 use nix::sys::stat::Mode;
 
 use pbs_datastore::catalog::CatalogWriter;
-
+use pbs_tools::sync::StdChannelWriter;
-use crate::tools::{
+use pbs_tools::tokio::TokioWriterAdapter;
-    StdChannelWriter,
-    TokioWriterAdapter,
-};
 
 /// Stream implementation to encode and upload .pxar archives.
 ///

src/client/task_log.rs

@@ -7,15 +7,10 @@ use futures::*;
 
 use proxmox::api::cli::format_and_print_result;
 
-use super::HttpClient;
+use pbs_tools::percent_encoding::percent_encode_component;
-use crate::{
-    server::{
-        worker_is_active_local,
-        UPID,
-    },
-    tools,
-};
 
+use super::HttpClient;
+use crate::server::{UPID, worker_is_active_local};
 
 /// Display task log on console
 ///
@@ -54,13 +49,13 @@ pub async fn display_task_log(
 
             let abort = abort_count.load(Ordering::Relaxed);
             if abort > 0 {
-                let path = format!("api2/json/nodes/localhost/tasks/{}", tools::percent_encode_component(upid_str));
+                let path = format!("api2/json/nodes/localhost/tasks/{}", percent_encode_component(upid_str));
                 let _ = client.delete(&path, None).await?;
             }
 
             let param = json!({ "start": start, "limit": limit, "test-status": true });
 
-            let path = format!("api2/json/nodes/localhost/tasks/{}/log", tools::percent_encode_component(upid_str));
+            let path = format!("api2/json/nodes/localhost/tasks/{}/log", percent_encode_component(upid_str));
             let result = client.get(&path, Some(param)).await?;
 
             let active = result["active"].as_bool().unwrap();

src/client/vsock_client.rs

@@ -1,10 +1,8 @@
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
 use anyhow::{bail, format_err, Error};
 use futures::*;
-
-use core::task::Context;
-use std::pin::Pin;
-use std::task::Poll;
-
 use http::Uri;
 use http::{Request, Response};
 use hyper::client::connect::{Connected, Connection};

src/tools/mod.rs

@@ -12,7 +12,6 @@ use std::path::Path;
 use anyhow::{bail, format_err, Error};
 use serde_json::Value;
 use openssl::hash::{hash, DigestBytes, MessageDigest};
-use percent_encoding::{utf8_percent_encode, AsciiSet};
 
 pub use proxmox::tools::fd::Fd;
 use proxmox::tools::fs::{create_path, CreateOptions};
@@ -33,7 +32,6 @@ pub use pbs_tools::process_locker::{
 pub mod acl;
 pub mod apt;
 pub mod async_io;
-pub mod cert;
 pub mod compression;
 pub mod config;
 pub mod cpio;
@@ -67,17 +65,10 @@ pub use wrapped_reader_stream::{AsyncReaderStream, StdChannelStream, WrappedRead
 mod async_channel_writer;
 pub use async_channel_writer::AsyncChannelWriter;
 
-mod std_channel_writer;
-pub use std_channel_writer::StdChannelWriter;
-
-mod tokio_writer_adapter;
-pub use tokio_writer_adapter::TokioWriterAdapter;
-
 mod file_logger;
 pub use file_logger::{FileLogger, FileLogOptions};
 
-mod broadcast_future;
+pub use pbs_tools::broadcast_future::{BroadcastData, BroadcastFuture};
-pub use broadcast_future::{BroadcastData, BroadcastFuture};
 
 /// The `BufferedRead` trait provides a single function
 /// `buffered_read`. It returns a reference to an internal buffer. The
@@ -235,11 +226,6 @@ pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
     None
 }
 
-/// percent encode a url component
-pub fn percent_encode_component(comp: &str) -> String {
-    utf8_percent_encode(comp, percent_encoding::NON_ALPHANUMERIC).to_string()
-}
-
 /// Detect modified configuration files
 ///
 /// This function fails with a reasonable error message if checksums do not match.
@@ -355,22 +341,6 @@ pub fn pbs_simple_http(proxy_config: Option<ProxyConfig>) -> SimpleHttp {
     SimpleHttp::with_options(options)
 }
 
-/// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}`
-pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e
-    // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above)
-    .add(0x20)
-    .add(0x7f)
-    // the DEFAULT_ENCODE_SET added:
-    .add(b' ')
-    .add(b'"')
-    .add(b'#')
-    .add(b'<')
-    .add(b'>')
-    .add(b'`')
-    .add(b'?')
-    .add(b'{')
-    .add(b'}');
-
 /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a
 /// `#`).
 pub fn file_get_non_comment_lines<P: AsRef<Path>>(