diff --git a/Cargo.toml b/Cargo.toml index 71deacd4..9c41605a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,6 @@ endian_trait = { version = "0.6", features = ["arrays"] } env_logger = "0.7" flate2 = "1.0" anyhow = "1.0" -foreign-types = "0.3" thiserror = "1.0" futures = "0.3" h2 = { version = "0.3", features = [ "stream" ] } diff --git a/pbs-tools/Cargo.toml b/pbs-tools/Cargo.toml index a3aa81c1..ab80666c 100644 --- a/pbs-tools/Cargo.toml +++ b/pbs-tools/Cargo.toml @@ -9,6 +9,9 @@ description = "common tools used throughout pbs" [dependencies] anyhow = "1.0" base64 = "0.12" +foreign-types = "0.3" +futures = "0.3" +lazy_static = "1.4" libc = "0.2" nix = "0.19.1" nom = "5.1" @@ -17,6 +20,10 @@ percent-encoding = "2.1" regex = "1.2" serde = "1.0" serde_json = "1.0" +# rt-multi-thread is required for block_in_place +tokio = { version = "1.6", features = [ "rt", "rt-multi-thread", "sync" ] } url = "2.1" proxmox = { version = "0.11.5", default-features = false, features = [] } + +pbs-buildcfg = { path = "../pbs-buildcfg" } diff --git a/pbs-tools/src/auth.rs b/pbs-tools/src/auth.rs new file mode 100644 index 00000000..6e605dd8 --- /dev/null +++ b/pbs-tools/src/auth.rs @@ -0,0 +1,26 @@ +//! Helpers for authentication used by both client and server. + +use anyhow::Error; +use lazy_static::lazy_static; +use openssl::pkey::{PKey, Private}; +use openssl::rsa::Rsa; + +use proxmox::tools::fs::file_get_contents; + +use pbs_buildcfg::configdir; + +fn load_private_auth_key() -> Result, Error> { + let pem = file_get_contents(configdir!("/authkey.key"))?; + let rsa = Rsa::private_key_from_pem(&pem)?; + let key = PKey::from_rsa(rsa)?; + + Ok(key) +} + +pub fn private_auth_key() -> &'static PKey { + lazy_static! { + static ref KEY: PKey = load_private_auth_key().unwrap(); + } + + &KEY +} diff --git a/src/tools/broadcast_future.rs b/pbs-tools/src/broadcast_future.rs similarity index 100% rename from src/tools/broadcast_future.rs rename to pbs-tools/src/broadcast_future.rs diff --git a/src/tools/cert.rs b/pbs-tools/src/cert.rs similarity index 100% rename from src/tools/cert.rs rename to pbs-tools/src/cert.rs diff --git a/pbs-tools/src/lib.rs b/pbs-tools/src/lib.rs index 72b0e9fd..c9d95dd9 100644 --- a/pbs-tools/src/lib.rs +++ b/pbs-tools/src/lib.rs @@ -1,12 +1,18 @@ +pub mod auth; pub mod borrow; +pub mod broadcast_future; +pub mod cert; pub mod format; pub mod fs; pub mod json; pub mod nom; +pub mod percent_encoding; pub mod process_locker; pub mod sha; pub mod str; +pub mod sync; pub mod ticket; +pub mod tokio; mod command; pub use command::{command_output, command_output_as_string, run_command}; diff --git a/pbs-tools/src/percent_encoding.rs b/pbs-tools/src/percent_encoding.rs new file mode 100644 index 00000000..afe011e2 --- /dev/null +++ b/pbs-tools/src/percent_encoding.rs @@ -0,0 +1,22 @@ +use percent_encoding::{utf8_percent_encode, AsciiSet}; + +/// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}` +pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e + // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above) + .add(0x20) + .add(0x7f) + // the DEFAULT_ENCODE_SET added: + .add(b' ') + .add(b'"') + .add(b'#') + .add(b'<') + .add(b'>') + .add(b'`') + .add(b'?') + .add(b'{') + .add(b'}'); + +/// percent encode a url component +pub fn percent_encode_component(comp: &str) -> String { + utf8_percent_encode(comp, percent_encoding::NON_ALPHANUMERIC).to_string() +} diff --git a/pbs-tools/src/sync/mod.rs b/pbs-tools/src/sync/mod.rs new file mode 100644 index 00000000..fdd5eda4 --- /dev/null +++ b/pbs-tools/src/sync/mod.rs @@ -0,0 +1,2 @@ +mod std_channel_writer; +pub use std_channel_writer::StdChannelWriter; diff --git a/src/tools/std_channel_writer.rs b/pbs-tools/src/sync/std_channel_writer.rs similarity index 100% rename from src/tools/std_channel_writer.rs rename to pbs-tools/src/sync/std_channel_writer.rs diff --git a/pbs-tools/src/tokio/mod.rs b/pbs-tools/src/tokio/mod.rs new file mode 100644 index 00000000..43fc107b --- /dev/null +++ b/pbs-tools/src/tokio/mod.rs @@ -0,0 +1,2 @@ +pub mod tokio_writer_adapter; +pub use tokio_writer_adapter::TokioWriterAdapter; diff --git a/src/tools/tokio_writer_adapter.rs b/pbs-tools/src/tokio/tokio_writer_adapter.rs similarity index 100% rename from src/tools/tokio_writer_adapter.rs rename to pbs-tools/src/tokio/tokio_writer_adapter.rs diff --git a/src/api2/access/mod.rs b/src/api2/access/mod.rs index c6bfbb9e..32dfe9de 100644 --- a/src/api2/access/mod.rs +++ b/src/api2/access/mod.rs @@ -11,6 +11,7 @@ use proxmox::api::{api, Permission, RpcEnvironment}; use proxmox::{http_err, list_subdirs_api_method}; use proxmox::{identity, sortable}; +use pbs_tools::auth::private_auth_key; use pbs_tools::ticket::{self, Empty, Ticket}; use crate::api2::types::*; diff --git a/src/api2/access/openid.rs b/src/api2/access/openid.rs index a778aa2a..783864cc 100644 --- a/src/api2/access/openid.rs +++ b/src/api2/access/openid.rs @@ -14,6 +14,7 @@ use proxmox::tools::fs::open_file_locked; use proxmox_openid::{OpenIdAuthenticator, OpenIdConfig}; use pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M; +use pbs_tools::auth::private_auth_key; use pbs_tools::ticket::Ticket; use crate::server::ticket::ApiTicket; diff --git a/src/api2/node/certificates.rs b/src/api2/node/certificates.rs index 34842673..b249b22d 100644 --- a/src/api2/node/certificates.rs +++ b/src/api2/node/certificates.rs @@ -12,6 +12,7 @@ use proxmox::api::{api, Permission, Router, RpcEnvironment}; use proxmox::list_subdirs_api_method; use pbs_buildcfg::configdir; +use pbs_tools::cert; use crate::acme::AcmeClient; use crate::api2::types::Authid; @@ -20,7 +21,6 @@ use crate::api2::types::AcmeDomain; use crate::config::acl::PRIV_SYS_MODIFY; use crate::config::node::NodeConfig; use crate::server::WorkerTask; -use crate::tools::cert; pub const ROUTER: Router = Router::new() .get(&list_subdirs_api_method!(SUBDIRS)) diff --git a/src/api2/node/mod.rs b/src/api2/node/mod.rs index 208cbf98..b9980ef7 100644 --- a/src/api2/node/mod.rs +++ b/src/api2/node/mod.rs @@ -20,6 +20,7 @@ use proxmox::list_subdirs_api_method; use proxmox_http::websocket::WebSocket; use proxmox::{identity, sortable}; +use pbs_tools::auth::private_auth_key; use pbs_tools::ticket::{self, Empty, Ticket}; use crate::api2::types::*; @@ -121,7 +122,7 @@ async fn termproxy( let ticket = Ticket::new(ticket::TERM_PREFIX, &Empty)? .sign( - crate::auth_helpers::private_auth_key(), + private_auth_key(), Some(&tools::ticket::term_aad(&userid, &path, port)), )?; diff --git a/src/api2/node/status.rs b/src/api2/node/status.rs index a82c0c8a..12f6dc71 100644 --- a/src/api2/node/status.rs +++ b/src/api2/node/status.rs @@ -8,9 +8,10 @@ use proxmox::sys::linux::procfs; use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission}; +use pbs_tools::cert::CertInfo; + use crate::api2::types::*; use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_POWER_MANAGEMENT}; -use crate::tools::cert::CertInfo; impl std::convert::From for NodeCpuInformation { fn from(info: procfs::ProcFsCPUInfo) -> Self { diff --git a/src/auth_helpers.rs b/src/auth_helpers.rs index 33d142b3..15e782a5 100644 --- a/src/auth_helpers.rs +++ b/src/auth_helpers.rs @@ -1,18 +1,16 @@ +use std::path::PathBuf; + use anyhow::{bail, format_err, Error}; use lazy_static::lazy_static; - -use openssl::rsa::{Rsa}; -use openssl::pkey::{PKey, Public, Private}; +use openssl::pkey::{PKey, Public}; +use openssl::rsa::Rsa; use openssl::sha; -use std::path::PathBuf; - use proxmox::tools::fs::{file_get_contents, replace_file, CreateOptions}; use proxmox::try_block; use pbs_buildcfg::configdir; - -use crate::api2::types::Userid; +use pbs_api_types::Userid; fn compute_csrf_secret_digest( timestamp: i64, @@ -155,24 +153,6 @@ pub fn csrf_secret() -> &'static [u8] { &SECRET } -fn load_private_auth_key() -> Result, Error> { - - let pem = file_get_contents(configdir!("/authkey.key"))?; - let rsa = Rsa::private_key_from_pem(&pem)?; - let key = PKey::from_rsa(rsa)?; - - Ok(key) -} - -pub fn private_auth_key() -> &'static PKey { - - lazy_static! { - static ref KEY: PKey = load_private_auth_key().unwrap(); - } - - &KEY -} - fn load_public_auth_key() -> Result, Error> { let pem = file_get_contents(configdir!("/authkey.pub"))?; diff --git a/src/bin/proxmox-backup-api.rs b/src/bin/proxmox-backup-api.rs index a8fbbadb..75104205 100644 --- a/src/bin/proxmox-backup-api.rs +++ b/src/bin/proxmox-backup-api.rs @@ -4,8 +4,8 @@ use futures::*; use proxmox::try_block; use proxmox::api::RpcEnvironmentType; -//use proxmox_backup::tools; -//use proxmox_backup::api_schema::config::*; +use pbs_tools::auth::private_auth_key; + use proxmox_backup::server::{ self, auth::default_api_auth, diff --git a/src/bin/proxmox-backup-client.rs b/src/bin/proxmox-backup-client.rs index a5810538..3a8a42a0 100644 --- a/src/bin/proxmox-backup-client.rs +++ b/src/bin/proxmox-backup-client.rs @@ -28,12 +28,9 @@ use proxmox::{ use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation}; use pbs_datastore::catalog::BackupCatalogWriter; +use pbs_tools::sync::StdChannelWriter; +use pbs_tools::tokio::TokioWriterAdapter; -use proxmox_backup::tools::{ - self, - StdChannelWriter, - TokioWriterAdapter, -}; use proxmox_backup::api2::types::*; use proxmox_backup::api2::version; use proxmox_backup::client::*; @@ -64,6 +61,7 @@ use proxmox_backup::backup::{ Shell, PruneOptions, }; +use proxmox_backup::tools; mod proxmox_backup_client; use proxmox_backup_client::*; diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs index 79f80513..228d7ed1 100644 --- a/src/bin/proxmox-backup-manager.rs +++ b/src/bin/proxmox-backup-manager.rs @@ -6,6 +6,8 @@ use serde_json::{json, Value}; use proxmox::api::{api, cli::*, RpcEnvironment}; +use pbs_tools::percent_encoding::percent_encode_component; + use proxmox_backup::tools; use proxmox_backup::config; use proxmox_backup::api2::{self, types::* }; @@ -188,7 +190,7 @@ async fn task_stop(param: Value) -> Result { let mut client = connect_to_localhost()?; - let path = format!("api2/json/nodes/localhost/tasks/{}", tools::percent_encode_component(upid_str)); + let path = format!("api2/json/nodes/localhost/tasks/{}", percent_encode_component(upid_str)); let _ = client.delete(&path, None).await?; Ok(Value::Null) diff --git a/src/bin/proxmox_backup_client/task.rs b/src/bin/proxmox_backup_client/task.rs index e7683932..e6fcc74e 100644 --- a/src/bin/proxmox_backup_client/task.rs +++ b/src/bin/proxmox_backup_client/task.rs @@ -3,6 +3,8 @@ use serde_json::{json, Value}; use proxmox::api::{api, cli::*}; +use pbs_tools::percent_encoding::percent_encode_component; + use proxmox_backup::tools; use proxmox_backup::client::*; @@ -125,7 +127,7 @@ async fn task_stop(param: Value) -> Result { let mut client = connect(&repo)?; - let path = format!("api2/json/nodes/localhost/tasks/{}", tools::percent_encode_component(upid_str)); + let path = format!("api2/json/nodes/localhost/tasks/{}", percent_encode_component(upid_str)); let _ = client.delete(&path, None).await?; Ok(Value::Null) diff --git a/src/bin/proxmox_backup_manager/cert.rs b/src/bin/proxmox_backup_manager/cert.rs index 845c8edc..c570572c 100644 --- a/src/bin/proxmox_backup_manager/cert.rs +++ b/src/bin/proxmox_backup_manager/cert.rs @@ -2,9 +2,10 @@ use anyhow::{bail, Error}; use proxmox::api::{api, cli::*}; +use pbs_tools::cert::CertInfo; + use proxmox_backup::config; use proxmox_backup::auth_helpers::*; -use proxmox_backup::tools::cert::CertInfo; #[api] /// Display node certificate information. diff --git a/src/client/backup_writer.rs b/src/client/backup_writer.rs index 5ab38b77..8b3ddefa 100644 --- a/src/client/backup_writer.rs +++ b/src/client/backup_writer.rs @@ -14,10 +14,15 @@ use tokio_stream::wrappers::ReceiverStream; use proxmox::tools::digest_to_hex; +use pbs_datastore::{CATALOG_NAME, CryptConfig}; +use pbs_datastore::data_blob::{ChunkInfo, DataBlob, DataChunkBuilder}; +use pbs_datastore::dynamic_index::DynamicIndexReader; +use pbs_datastore::fixed_index::FixedIndexReader; +use pbs_datastore::index::IndexFile; +use pbs_datastore::manifest::{ArchiveType, BackupManifest, MANIFEST_BLOB_NAME}; use pbs_tools::format::HumanByte; use super::merge_known_chunks::{MergeKnownChunks, MergedChunkInfo}; -use crate::backup::*; use super::{H2Client, HttpClient}; @@ -283,7 +288,7 @@ impl BackupWriter { if let Some(manifest) = options.previous_manifest { // try, but ignore errors - match archive_type(archive_name) { + match ArchiveType::from_path(archive_name) { Ok(ArchiveType::FixedIndex) => { let _ = self .download_previous_fixed_index( diff --git a/src/client/http_client.rs b/src/client/http_client.rs index d19fa5c2..a83b8d3c 100644 --- a/src/client/http_client.rs +++ b/src/client/http_client.rs @@ -24,15 +24,13 @@ use proxmox_http::client::HttpsConnector; use proxmox_http::uri::build_authority; use pbs_api_types::{Authid, Userid}; +use pbs_tools::broadcast_future::BroadcastFuture; use pbs_tools::json::json_object_to_query; use pbs_tools::ticket; +use pbs_tools::percent_encoding::DEFAULT_ENCODE_SET; use super::pipe_to_stream::PipeToSendStream; -use crate::tools::{ - BroadcastFuture, - DEFAULT_ENCODE_SET, - PROXMOX_BACKUP_TCP_KEEPALIVE_TIME, -}; +use super::PROXMOX_BACKUP_TCP_KEEPALIVE_TIME; /// Timeout used for several HTTP operations that are expected to finish quickly but may block in /// certain error conditions. Keep it generous, to avoid false-positive under high load. diff --git a/src/client/mod.rs b/src/client/mod.rs index 8a4e4556..8ff00018 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -7,11 +7,8 @@ use anyhow::Error; use pbs_api_types::{Authid, Userid}; use pbs_tools::ticket::Ticket; - -use crate::{ - tools::cert::CertInfo, - auth_helpers::private_auth_key, -}; +use pbs_tools::cert::CertInfo; +use pbs_tools::auth::private_auth_key; mod merge_known_chunks; pub mod pipe_to_stream; @@ -43,6 +40,8 @@ pub use backup_repo::*; mod backup_specification; pub use backup_specification::*; +pub const PROXMOX_BACKUP_TCP_KEEPALIVE_TIME: u32 = 120; + /// Connect to localhost:8007 as root@pam /// /// This automatically creates a ticket if run as 'root' user. diff --git a/src/client/pxar_backup_stream.rs b/src/client/pxar_backup_stream.rs index 86bc8583..d39eb6c4 100644 --- a/src/client/pxar_backup_stream.rs +++ b/src/client/pxar_backup_stream.rs @@ -13,11 +13,8 @@ use nix::fcntl::OFlag; use nix::sys::stat::Mode; use pbs_datastore::catalog::CatalogWriter; - -use crate::tools::{ - StdChannelWriter, - TokioWriterAdapter, -}; +use pbs_tools::sync::StdChannelWriter; +use pbs_tools::tokio::TokioWriterAdapter; /// Stream implementation to encode and upload .pxar archives. /// diff --git a/src/client/task_log.rs b/src/client/task_log.rs index 695e6f3b..6350e83a 100644 --- a/src/client/task_log.rs +++ b/src/client/task_log.rs @@ -7,15 +7,10 @@ use futures::*; use proxmox::api::cli::format_and_print_result; -use super::HttpClient; -use crate::{ - server::{ - worker_is_active_local, - UPID, - }, - tools, -}; +use pbs_tools::percent_encoding::percent_encode_component; +use super::HttpClient; +use crate::server::{UPID, worker_is_active_local}; /// Display task log on console /// @@ -54,13 +49,13 @@ pub async fn display_task_log( let abort = abort_count.load(Ordering::Relaxed); if abort > 0 { - let path = format!("api2/json/nodes/localhost/tasks/{}", tools::percent_encode_component(upid_str)); + let path = format!("api2/json/nodes/localhost/tasks/{}", percent_encode_component(upid_str)); let _ = client.delete(&path, None).await?; } let param = json!({ "start": start, "limit": limit, "test-status": true }); - let path = format!("api2/json/nodes/localhost/tasks/{}/log", tools::percent_encode_component(upid_str)); + let path = format!("api2/json/nodes/localhost/tasks/{}/log", percent_encode_component(upid_str)); let result = client.get(&path, Some(param)).await?; let active = result["active"].as_bool().unwrap(); diff --git a/src/client/vsock_client.rs b/src/client/vsock_client.rs index d735b6ea..3f0f3738 100644 --- a/src/client/vsock_client.rs +++ b/src/client/vsock_client.rs @@ -1,10 +1,8 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; + use anyhow::{bail, format_err, Error}; use futures::*; - -use core::task::Context; -use std::pin::Pin; -use std::task::Poll; - use http::Uri; use http::{Request, Response}; use hyper::client::connect::{Connected, Connection}; diff --git a/src/tools/mod.rs b/src/tools/mod.rs index 900f33c0..658c7014 100644 --- a/src/tools/mod.rs +++ b/src/tools/mod.rs @@ -12,7 +12,6 @@ use std::path::Path; use anyhow::{bail, format_err, Error}; use serde_json::Value; use openssl::hash::{hash, DigestBytes, MessageDigest}; -use percent_encoding::{utf8_percent_encode, AsciiSet}; pub use proxmox::tools::fd::Fd; use proxmox::tools::fs::{create_path, CreateOptions}; @@ -33,7 +32,6 @@ pub use pbs_tools::process_locker::{ pub mod acl; pub mod apt; pub mod async_io; -pub mod cert; pub mod compression; pub mod config; pub mod cpio; @@ -67,17 +65,10 @@ pub use wrapped_reader_stream::{AsyncReaderStream, StdChannelStream, WrappedRead mod async_channel_writer; pub use async_channel_writer::AsyncChannelWriter; -mod std_channel_writer; -pub use std_channel_writer::StdChannelWriter; - -mod tokio_writer_adapter; -pub use tokio_writer_adapter::TokioWriterAdapter; - mod file_logger; pub use file_logger::{FileLogger, FileLogOptions}; -mod broadcast_future; -pub use broadcast_future::{BroadcastData, BroadcastFuture}; +pub use pbs_tools::broadcast_future::{BroadcastData, BroadcastFuture}; /// The `BufferedRead` trait provides a single function /// `buffered_read`. It returns a reference to an internal buffer. The @@ -235,11 +226,6 @@ pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option { None } -/// percent encode a url component -pub fn percent_encode_component(comp: &str) -> String { - utf8_percent_encode(comp, percent_encoding::NON_ALPHANUMERIC).to_string() -} - /// Detect modified configuration files /// /// This function fails with a reasonable error message if checksums do not match. @@ -355,22 +341,6 @@ pub fn pbs_simple_http(proxy_config: Option) -> SimpleHttp { SimpleHttp::with_options(options) } -/// This used to be: `SIMPLE_ENCODE_SET` plus space, `"`, `#`, `<`, `>`, backtick, `?`, `{`, `}` -pub const DEFAULT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS // 0..1f and 7e - // The SIMPLE_ENCODE_SET adds space and anything >= 0x7e (7e itself is already included above) - .add(0x20) - .add(0x7f) - // the DEFAULT_ENCODE_SET added: - .add(b' ') - .add(b'"') - .add(b'#') - .add(b'<') - .add(b'>') - .add(b'`') - .add(b'?') - .add(b'{') - .add(b'}'); - /// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a /// `#`). pub fn file_get_non_comment_lines>(