move client to pbs-client subcrate

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>

Cargo.toml

@@ -22,6 +22,7 @@ exclude = [ "build", "debian", "tests/catar_data/test_symlink/symlink1"]
 [workspace]
 members = [
     "pbs-buildcfg",
+    "pbs-client",
     "pbs-datastore",
     "pbs-runtime",
     "pbs-systemd",
@@ -95,6 +96,7 @@ proxmox-openid = "0.6.0"
 
 pbs-api-types = { path = "pbs-api-types" }
 pbs-buildcfg = { path = "pbs-buildcfg" }
+pbs-client = { path = "pbs-client" }
 pbs-datastore = { path = "pbs-datastore" }
 pbs-runtime = { path = "pbs-runtime" }
 pbs-systemd = { path = "pbs-systemd" }

Makefile

@@ -33,6 +33,7 @@ RESTORE_BIN := \
 SUBCRATES := \
 	pbs-api-types \
 	pbs-buildcfg \
+	pbs-client \
 	pbs-datastore \
 	pbs-runtime \
 	pbs-systemd \

examples/download-speed.rs

@@ -2,8 +2,8 @@ use std::io::Write;
 
 use anyhow::{Error};
 
-use proxmox_backup::api2::types::Authid;
+use pbs_api_types::Authid;
-use proxmox_backup::client::{HttpClient, HttpClientOptions, BackupReader};
+use pbs_client::{HttpClient, HttpClientOptions, BackupReader};
 
 pub struct DummyWriter {
     bytes: usize,

examples/upload-speed.rs

@@ -1,7 +1,7 @@
 use anyhow::{Error};
 
-use proxmox_backup::api2::types::Authid;
+use pbs_client::{HttpClient, HttpClientOptions, BackupWriter};
-use proxmox_backup::client::*;
+use pbs_api_types::Authid;
 
 async fn upload_speed() -> Result<f64, Error> {
 

pbs-api-types/src/lib.rs

@@ -40,6 +40,13 @@ pub use userid::{Tokenname, TokennameRef};
 pub use userid::{Username, UsernameRef};
 pub use userid::{PROXMOX_GROUP_ID_SCHEMA, PROXMOX_TOKEN_ID_SCHEMA, PROXMOX_TOKEN_NAME_SCHEMA};
 
+#[macro_use]
+mod user;
+pub use user::{ApiToken, User, UserWithTokens};
+pub use user::{
+    EMAIL_SCHEMA, ENABLE_USER_SCHEMA, EXPIRE_USER_SCHEMA, FIRST_NAME_SCHEMA, LAST_NAME_SCHEMA,
+};
+
 pub mod upid;
 pub use upid::UPID;
 
@@ -146,33 +153,31 @@ pub const CERT_FINGERPRINT_SHA256_SCHEMA: Schema =
         .format(&FINGERPRINT_SHA256_FORMAT)
         .schema();
 
-pub const PRUNE_SCHEMA_KEEP_DAILY: Schema = IntegerSchema::new(
+pub const PRUNE_SCHEMA_KEEP_DAILY: Schema = IntegerSchema::new("Number of daily backups to keep.")
-    "Number of daily backups to keep.")
     .minimum(1)
     .schema();
 
-pub const PRUNE_SCHEMA_KEEP_HOURLY: Schema = IntegerSchema::new(
+pub const PRUNE_SCHEMA_KEEP_HOURLY: Schema =
-    "Number of hourly backups to keep.")
+    IntegerSchema::new("Number of hourly backups to keep.")
         .minimum(1)
         .schema();
 
-pub const PRUNE_SCHEMA_KEEP_LAST: Schema = IntegerSchema::new(
+pub const PRUNE_SCHEMA_KEEP_LAST: Schema = IntegerSchema::new("Number of backups to keep.")
-    "Number of backups to keep.")
     .minimum(1)
     .schema();
 
-pub const PRUNE_SCHEMA_KEEP_MONTHLY: Schema = IntegerSchema::new(
+pub const PRUNE_SCHEMA_KEEP_MONTHLY: Schema =
-    "Number of monthly backups to keep.")
+    IntegerSchema::new("Number of monthly backups to keep.")
         .minimum(1)
         .schema();
 
-pub const PRUNE_SCHEMA_KEEP_WEEKLY: Schema = IntegerSchema::new(
+pub const PRUNE_SCHEMA_KEEP_WEEKLY: Schema =
-    "Number of weekly backups to keep.")
+    IntegerSchema::new("Number of weekly backups to keep.")
         .minimum(1)
         .schema();
 
-pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema = IntegerSchema::new(
+pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema =
-    "Number of yearly backups to keep.")
+    IntegerSchema::new("Number of yearly backups to keep.")
         .minimum(1)
         .schema();
 
@@ -186,6 +191,14 @@ pub const SINGLE_LINE_COMMENT_SCHEMA: Schema = StringSchema::new("Comment (singl
     .format(&SINGLE_LINE_COMMENT_FORMAT)
     .schema();
 
+pub const PROXMOX_CONFIG_DIGEST_SCHEMA: Schema = StringSchema::new(
+    "Prevent changes if current configuration file has different \
+    SHA256 digest. This can be used to prevent concurrent \
+    modifications.",
+)
+.format(&PVE_CONFIG_DIGEST_FORMAT)
+.schema();
+
 pub const BACKUP_ID_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&BACKUP_ID_REGEX);
 
 /// API schema format definition for repository URLs

pbs-api-types/src/user.rs

@@ -0,0 +1,205 @@
+use serde::{Deserialize, Serialize};
+
+use proxmox::api::api;
+use proxmox::api::schema::{BooleanSchema, IntegerSchema, Schema, StringSchema};
+
+use super::{SINGLE_LINE_COMMENT_FORMAT, SINGLE_LINE_COMMENT_SCHEMA};
+use super::userid::{Authid, Userid, PROXMOX_TOKEN_ID_SCHEMA};
+
+pub const ENABLE_USER_SCHEMA: Schema = BooleanSchema::new(
+    "Enable the account (default). You can set this to '0' to disable the account.")
+    .default(true)
+    .schema();
+
+pub const EXPIRE_USER_SCHEMA: Schema = IntegerSchema::new(
+    "Account expiration date (seconds since epoch). '0' means no expiration date.")
+    .default(0)
+    .minimum(0)
+    .schema();
+
+pub const FIRST_NAME_SCHEMA: Schema = StringSchema::new("First name.")
+    .format(&SINGLE_LINE_COMMENT_FORMAT)
+    .min_length(2)
+    .max_length(64)
+    .schema();
+
+pub const LAST_NAME_SCHEMA: Schema = StringSchema::new("Last name.")
+    .format(&SINGLE_LINE_COMMENT_FORMAT)
+    .min_length(2)
+    .max_length(64)
+    .schema();
+
+pub const EMAIL_SCHEMA: Schema = StringSchema::new("E-Mail Address.")
+    .format(&SINGLE_LINE_COMMENT_FORMAT)
+    .min_length(2)
+    .max_length(64)
+    .schema();
+
+#[api(
+    properties: {
+        userid: {
+            type: Userid,
+        },
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+        enable: {
+            optional: true,
+            schema: ENABLE_USER_SCHEMA,
+        },
+        expire: {
+            optional: true,
+            schema: EXPIRE_USER_SCHEMA,
+        },
+        firstname: {
+            optional: true,
+            schema: FIRST_NAME_SCHEMA,
+        },
+        lastname: {
+            schema: LAST_NAME_SCHEMA,
+            optional: true,
+         },
+        email: {
+            schema: EMAIL_SCHEMA,
+            optional: true,
+        },
+        tokens: {
+            type: Array,
+            optional: true,
+            description: "List of user's API tokens.",
+            items: {
+                type: ApiToken
+            },
+        },
+    }
+)]
+#[derive(Serialize,Deserialize)]
+/// User properties with added list of ApiTokens
+pub struct UserWithTokens {
+    pub userid: Userid,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub comment: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub enable: Option<bool>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub expire: Option<i64>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub firstname: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub lastname: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub email: Option<String>,
+    #[serde(skip_serializing_if="Vec::is_empty", default)]
+    pub tokens: Vec<ApiToken>,
+}
+
+#[api(
+    properties: {
+        tokenid: {
+            schema: PROXMOX_TOKEN_ID_SCHEMA,
+        },
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+        enable: {
+            optional: true,
+            schema: ENABLE_USER_SCHEMA,
+        },
+        expire: {
+            optional: true,
+            schema: EXPIRE_USER_SCHEMA,
+        },
+    }
+)]
+#[derive(Serialize,Deserialize)]
+/// ApiToken properties.
+pub struct ApiToken {
+    pub tokenid: Authid,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub comment: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub enable: Option<bool>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub expire: Option<i64>,
+}
+
+impl ApiToken {
+    pub fn is_active(&self) -> bool {
+        if !self.enable.unwrap_or(true) {
+            return false;
+        }
+        if let Some(expire) = self.expire {
+            let now =  proxmox::tools::time::epoch_i64();
+            if expire > 0 && expire <= now {
+                return false;
+            }
+        }
+        true
+    }
+}
+
+#[api(
+    properties: {
+        userid: {
+            type: Userid,
+        },
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+        enable: {
+            optional: true,
+            schema: ENABLE_USER_SCHEMA,
+        },
+        expire: {
+            optional: true,
+            schema: EXPIRE_USER_SCHEMA,
+        },
+        firstname: {
+            optional: true,
+            schema: FIRST_NAME_SCHEMA,
+        },
+        lastname: {
+            schema: LAST_NAME_SCHEMA,
+            optional: true,
+         },
+        email: {
+            schema: EMAIL_SCHEMA,
+            optional: true,
+        },
+    }
+)]
+#[derive(Serialize,Deserialize)]
+/// User properties.
+pub struct User {
+    pub userid: Userid,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub comment: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub enable: Option<bool>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub expire: Option<i64>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub firstname: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub lastname: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub email: Option<String>,
+}
+
+impl User {
+    pub fn is_active(&self) -> bool {
+        if !self.enable.unwrap_or(true) {
+            return false;
+        }
+        if let Some(expire) = self.expire {
+            let now =  proxmox::tools::time::epoch_i64();
+            if expire > 0 && expire <= now {
+                return false;
+            }
+        }
+        true
+    }
+}

pbs-api-types/src/userid.rs

@@ -98,7 +98,6 @@ pub const PROXMOX_AUTH_REALM_STRING_SCHEMA: StringSchema =
         .max_length(32);
 pub const PROXMOX_AUTH_REALM_SCHEMA: Schema = PROXMOX_AUTH_REALM_STRING_SCHEMA.schema();
 
-
 #[api(
     type: String,
     format: &PROXMOX_USER_NAME_FORMAT,

pbs-client/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "pbs-client"
+version = "0.1.0"
+authors = ["Wolfgang Bumiller <w.bumiller@proxmox.com>"]
+edition = "2018"
+description = "The main proxmox backup client crate"
+
+[dependencies]
+anyhow = "1.0"
+bitflags = "1.2.1"
+bytes = "1.0"
+futures = "0.3"
+h2 = { version = "0.3", features = [ "stream" ] }
+http = "0.2"
+hyper = { version = "0.14", features = [ "full" ] }
+lazy_static = "1.4"
+libc = "0.2"
+nix = "0.19.1"
+openssl = "0.10"
+percent-encoding = "2.1"
+pin-project = "1.0"
+regex = "1.2"
+rustyline = "7"
+serde_json = "1.0"
+tokio = { version = "1.6", features = [ "fs", "signal" ] }
+tokio-stream = "0.1.0"
+tower-service = "0.3.0"
+xdg = "2.2"
+
+pathpatterns = "0.1.2"
+proxmox = { version = "0.11.5", default-features = false, features = [ "cli" ] }
+proxmox-fuse = "0.1.1"
+proxmox-http = { version = "0.2.1", features = [ "client", "http-helpers", "websocket" ] }
+pxar = { version = "0.10.1", features = [ "tokio-io" ] }
+
+pbs-api-types = { path = "../pbs-api-types" }
+pbs-buildcfg = { path = "../pbs-buildcfg" }
+pbs-datastore = { path = "../pbs-datastore" }
+pbs-runtime = { path = "../pbs-runtime" }
+pbs-tools = { path = "../pbs-tools" }

pbs-client/src/backup_reader.rs

@@ -9,7 +9,7 @@ use serde_json::{json, Value};
 
 use proxmox::tools::digest_to_hex;
 
-use pbs_datastore::{CryptConfig, BackupManifest};
+use pbs_datastore::{PROXMOX_BACKUP_READER_PROTOCOL_ID_V1, CryptConfig, BackupManifest};
 use pbs_datastore::data_blob::DataBlob;
 use pbs_datastore::data_blob_reader::DataBlobReader;
 use pbs_datastore::dynamic_index::DynamicIndexReader;

pbs-client/src/backup_writer.rs

@@ -14,7 +14,7 @@ use tokio_stream::wrappers::ReceiverStream;
 
 use proxmox::tools::digest_to_hex;
 
-use pbs_datastore::{CATALOG_NAME, CryptConfig};
+use pbs_datastore::{CATALOG_NAME, PROXMOX_BACKUP_PROTOCOL_ID_V1, CryptConfig};
 use pbs_datastore::data_blob::{ChunkInfo, DataBlob, DataChunkBuilder};
 use pbs_datastore::dynamic_index::DynamicIndexReader;
 use pbs_datastore::fixed_index::FixedIndexReader;

pbs-client/src/catalog_shell.rs

@@ -18,13 +18,14 @@ use proxmox::api::cli::{self, CliCommand, CliCommandMap, CliHelper, CommandLineI
 use proxmox::tools::fs::{create_path, CreateOptions};
 use pxar::{EntryKind, Metadata};
 
-use crate::backup::catalog::{self, DirEntryAttribute};
-use crate::pxar::fuse::{Accessor, FileEntry};
-use crate::pxar::Flags;
 use pbs_runtime::block_in_place;
-use crate::tools::ControlFlow;
+use pbs_datastore::catalog::{self, DirEntryAttribute};
+use pbs_tools::ops::ControlFlow;
 
-type CatalogReader = crate::backup::CatalogReader<std::fs::File>;
+use crate::pxar::Flags;
+use crate::pxar::fuse::{Accessor, FileEntry};
+
+type CatalogReader = pbs_datastore::catalog::CatalogReader<std::fs::File>;
 
 const MAX_SYMLINK_COUNT: usize = 40;
 
@@ -78,13 +79,13 @@ pub fn catalog_shell_cli() -> CommandLineInterface {
                 "restore-selected",
                 CliCommand::new(&API_METHOD_RESTORE_SELECTED_COMMAND)
                     .arg_param(&["target"])
-                    .completion_cb("target", crate::tools::complete_file_name),
+                    .completion_cb("target", pbs_tools::fs::complete_file_name),
             )
             .insert(
                 "restore",
                 CliCommand::new(&API_METHOD_RESTORE_COMMAND)
                     .arg_param(&["target"])
-                    .completion_cb("target", crate::tools::complete_file_name),
+                    .completion_cb("target", pbs_tools::fs::complete_file_name),
             )
             .insert(
                 "find",
@@ -985,7 +986,8 @@ impl Shell {
             .metadata()
             .clone();
 
-        let extractor = crate::pxar::extract::Extractor::new(rootdir, root_meta, true, Flags::DEFAULT);
+        let extractor =
+            crate::pxar::extract::Extractor::new(rootdir, root_meta, true, Flags::DEFAULT);
 
         let mut extractor = ExtractorState::new(
             &mut self.catalog,

pbs-client/src/lib.rs

@@ -10,6 +10,10 @@ use pbs_tools::ticket::Ticket;
 use pbs_tools::cert::CertInfo;
 use pbs_tools::auth::private_auth_key;
 
+pub mod catalog_shell;
+pub mod pxar;
+pub mod tools;
+
 mod merge_known_chunks;
 pub mod pipe_to_stream;
 

pbs-client/src/pxar/create.rs

@@ -23,14 +23,15 @@ use proxmox::c_str;
 use proxmox::sys::error::SysError;
 use proxmox::tools::fd::RawFdNum;
 use proxmox::tools::vec;
+use proxmox::tools::fd::Fd;
 
 use pbs_datastore::catalog::BackupCatalogWriter;
-use pbs_tools::fs;
+use pbs_tools::{acl, fs, xattr};
+use pbs_tools::str::strip_ascii_whitespace;
 
 use crate::pxar::metadata::errno_is_unsupported;
 use crate::pxar::Flags;
 use crate::pxar::tools::assert_single_path_component;
-use crate::tools::{acl, xattr, Fd};
 
 /// Pxar options for creating a pxar archive/stream
 #[derive(Default, Clone)]
@@ -362,7 +363,7 @@ impl Archiver {
                 }
             };
 
-            let line = crate::tools::strip_ascii_whitespace(&line);
+            let line = strip_ascii_whitespace(&line);
 
             if line.is_empty() || line[0] == b'#' {
                 continue;

pbs-client/src/pxar/extract.rs

@@ -27,12 +27,12 @@ use proxmox::tools::{
     io::{sparse_copy, sparse_copy_async},
 };
 
+use pbs_tools::zip::{ZipEncoder, ZipEntry};
+
 use crate::pxar::dir_stack::PxarDirStack;
 use crate::pxar::metadata;
 use crate::pxar::Flags;
 
-use crate::tools::zip::{ZipEncoder, ZipEntry};
-
 pub struct PxarExtractOptions<'a> {
     pub match_list: &'a[MatchEntry],
     pub extract_match_default: bool,
@@ -215,7 +215,7 @@ where
 }
 
 /// Common state for file extraction.
-pub(crate) struct Extractor {
+pub struct Extractor {
     feature_flags: Flags,
     allow_existing_dirs: bool,
     dir_stack: PxarDirStack,

pbs-client/src/pxar/fuse.rs

@@ -26,7 +26,7 @@ use pxar::accessor::{self, EntryRangeInfo, ReadAt};
 use proxmox_fuse::requests::{self, FuseRequest};
 use proxmox_fuse::{EntryParam, Fuse, ReplyBufState, Request, ROOT_ID};
 
-use crate::tools::xattr;
+use pbs_tools::xattr;
 
 /// We mark inodes for regular files this way so we know how to access them.
 const NON_DIRECTORY_INODE: u64 = 1u64 << 63;

pbs-client/src/pxar/metadata.rs

@@ -13,11 +13,10 @@ use proxmox::c_result;
 use proxmox::sys::error::SysError;
 use proxmox::tools::fd::RawFdNum;
 
-use pbs_tools::fs;
+use pbs_tools::{acl, fs, xattr};
 
 use crate::pxar::tools::perms_from_metadata;
 use crate::pxar::Flags;
-use crate::tools::{acl, xattr};
 
 //
 // utility functions

pbs-client/src/pxar/mod.rs

@@ -50,8 +50,8 @@
 pub(crate) mod create;
 pub(crate) mod dir_stack;
 pub(crate) mod extract;
-pub(crate) mod metadata;
 pub mod fuse;
+pub(crate) mod metadata;
 pub(crate) mod tools;
 
 mod flags;

pbs-client/src/tools/key_source.rs

@@ -10,7 +10,7 @@ use proxmox::api::schema::*;
 use proxmox::sys::linux::tty;
 use proxmox::tools::fs::file_get_contents;
 
-use proxmox_backup::backup::CryptMode;
+use pbs_api_types::CryptMode;
 
 pub const DEFAULT_ENCRYPTION_KEY_FILE_NAME: &str = "encryption-key.json";
 pub const DEFAULT_MASTER_PUBKEY_FILE_NAME: &str = "master-public.pem";

pbs-client/src/tools/mod.rs

@@ -10,14 +10,11 @@ use proxmox::{
     tools::fs::file_get_json,
 };
 
-use pbs_api_types::{BACKUP_REPO_URL, Authid};
+use pbs_api_types::{BACKUP_REPO_URL, Authid, UserWithTokens};
-use pbs_buildcfg;
 use pbs_datastore::BackupDir;
 use pbs_tools::json::json_object_to_query;
 
-use proxmox_backup::api2::access::user::UserWithTokens;
+use crate::{BackupRepository, HttpClient, HttpClientOptions};
-use proxmox_backup::client::{BackupRepository, HttpClient, HttpClientOptions};
-use proxmox_backup::tools;
 
 pub mod key_source;
 
@@ -342,7 +339,7 @@ pub fn complete_backup_source(arg: &str, param: &HashMap<String, String>) -> Vec
         return result;
     }
 
-    let files = tools::complete_file_name(data[1], param);
+    let files = pbs_tools::fs::complete_file_name(data[1], param);
 
     for file in files {
         result.push(format!("{}:{}", data[0], file));
@@ -375,15 +372,3 @@ pub fn place_xdg_file(
         .and_then(|base| base.place_config_file(file_name).map_err(Error::from))
         .with_context(|| format!("failed to place {} in xdg home", description))
 }
-
-/// Returns a runtime dir owned by the current user.
-/// Note that XDG_RUNTIME_DIR is not always available, especially for non-login users like
-/// "www-data", so we use a custom one in /run/proxmox-backup/<uid> instead.
-pub fn get_user_run_dir() -> Result<std::path::PathBuf, Error> {
-    let uid = nix::unistd::Uid::current();
-    let mut path: std::path::PathBuf = pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR.into();
-    path.push(uid.to_string());
-    tools::create_run_dir()?;
-    std::fs::create_dir_all(&path)?;
-    Ok(path)
-}

pbs-tools/Cargo.toml

@@ -9,6 +9,10 @@ description = "common tools used throughout pbs"
 [dependencies]
 anyhow = "1.0"
 base64 = "0.12"
+bytes = "1.0"
+crc32fast = "1"
+endian_trait = { version = "0.6", features = ["arrays"] }
+flate2 = "1.0"
 foreign-types = "0.3"
 futures = "0.3"
 lazy_static = "1.4"
@@ -21,9 +25,10 @@ regex = "1.2"
 serde = "1.0"
 serde_json = "1.0"
 # rt-multi-thread is required for block_in_place
-tokio = { version = "1.6", features = [ "rt", "rt-multi-thread", "sync" ] }
+tokio = { version = "1.6", features = [ "fs", "io-util", "rt", "rt-multi-thread", "sync" ] }
 url = "2.1"
+walkdir = "2"
 
-proxmox = { version = "0.11.5", default-features = false, features = [] }
+proxmox = { version = "0.11.5", default-features = false, features = [ "tokio" ] }
 
 pbs-buildcfg = { path = "../pbs-buildcfg" }

pbs-tools/src/compression.rs

@@ -0,0 +1,194 @@
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use anyhow::Error;
+use bytes::Bytes;
+use flate2::{Compress, Compression, FlushCompress};
+use futures::ready;
+use futures::stream::Stream;
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+
+use proxmox::io_format_err;
+use proxmox::tools::byte_buffer::ByteBuffer;
+
+const BUFFER_SIZE: usize = 8192;
+
+pub enum Level {
+    Fastest,
+    Best,
+    Default,
+    Precise(u32),
+}
+
+#[derive(Eq, PartialEq)]
+enum EncoderState {
+    Reading,
+    Writing,
+    Flushing,
+    Finished,
+}
+
+pub struct DeflateEncoder<T> {
+    inner: T,
+    compressor: Compress,
+    buffer: ByteBuffer,
+    input_buffer: Bytes,
+    state: EncoderState,
+}
+
+impl<T> DeflateEncoder<T> {
+    pub fn new(inner: T) -> Self {
+        Self::with_quality(inner, Level::Default)
+    }
+
+    pub fn with_quality(inner: T, level: Level) -> Self {
+        let level = match level {
+            Level::Fastest => Compression::fast(),
+            Level::Best => Compression::best(),
+            Level::Default => Compression::new(3),
+            Level::Precise(val) => Compression::new(val),
+        };
+
+        Self {
+            inner,
+            compressor: Compress::new(level, false),
+            buffer: ByteBuffer::with_capacity(BUFFER_SIZE),
+            input_buffer: Bytes::new(),
+            state: EncoderState::Reading,
+        }
+    }
+
+    pub fn total_in(&self) -> u64 {
+        self.compressor.total_in()
+    }
+
+    pub fn total_out(&self) -> u64 {
+        self.compressor.total_out()
+    }
+
+    pub fn into_inner(self) -> T {
+        self.inner
+    }
+
+    fn encode(
+        &mut self,
+        inbuf: &[u8],
+        flush: FlushCompress,
+    ) -> Result<(usize, flate2::Status), io::Error> {
+        let old_in = self.compressor.total_in();
+        let old_out = self.compressor.total_out();
+        let res = self
+            .compressor
+            .compress(&inbuf[..], self.buffer.get_free_mut_slice(), flush)?;
+        let new_in = (self.compressor.total_in() - old_in) as usize;
+        let new_out = (self.compressor.total_out() - old_out) as usize;
+        self.buffer.add_size(new_out);
+
+        Ok((new_in, res))
+    }
+}
+
+impl DeflateEncoder<Vec<u8>> {
+    // assume small files
+    pub async fn compress_vec<R>(&mut self, reader: &mut R, size_hint: usize) -> Result<(), Error>
+    where
+        R: AsyncRead + Unpin,
+    {
+        let mut buffer = Vec::with_capacity(size_hint);
+        reader.read_to_end(&mut buffer).await?;
+        self.inner.reserve(size_hint); // should be enough since we want smalller files
+        self.compressor.compress_vec(&buffer[..], &mut self.inner, FlushCompress::Finish)?;
+        Ok(())
+    }
+}
+
+impl<T: AsyncWrite + Unpin> DeflateEncoder<T> {
+    pub async fn compress<R>(&mut self, reader: &mut R) -> Result<(), Error>
+    where
+        R: AsyncRead + Unpin,
+    {
+        let mut buffer = ByteBuffer::with_capacity(BUFFER_SIZE);
+        let mut eof = false;
+        loop {
+            if !eof && !buffer.is_full() {
+                let read = buffer.read_from_async(reader).await?;
+                if read == 0 {
+                    eof = true;
+                }
+            }
+            let (read, _res) = self.encode(&buffer[..], FlushCompress::None)?;
+            buffer.consume(read);
+
+            self.inner.write_all(&self.buffer[..]).await?;
+            self.buffer.clear();
+
+            if buffer.is_empty() && eof {
+                break;
+            }
+        }
+
+        loop {
+            let (_read, res) = self.encode(&[][..], FlushCompress::Finish)?;
+            self.inner.write_all(&self.buffer[..]).await?;
+            self.buffer.clear();
+            if res == flate2::Status::StreamEnd {
+                break;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl<T, O> Stream for DeflateEncoder<T>
+where
+    T: Stream<Item = Result<O, io::Error>> + Unpin,
+    O: Into<Bytes>
+{
+    type Item = Result<Bytes, io::Error>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.get_mut();
+
+        loop {
+            match this.state {
+                EncoderState::Reading => {
+                    if let Some(res) = ready!(Pin::new(&mut this.inner).poll_next(cx)) {
+                        let buf = res?;
+                        this.input_buffer = buf.into();
+                        this.state = EncoderState::Writing;
+                    } else {
+                        this.state = EncoderState::Flushing;
+                    }
+                }
+                EncoderState::Writing => {
+                    if this.input_buffer.is_empty() {
+                        return Poll::Ready(Some(Err(io_format_err!("empty input during write"))));
+                    }
+                    let mut buf = this.input_buffer.split_off(0);
+                    let (read, res) = this.encode(&buf[..], FlushCompress::None)?;
+                    this.input_buffer = buf.split_off(read);
+                    if this.input_buffer.is_empty() {
+                        this.state = EncoderState::Reading;
+                    }
+                    if this.buffer.is_full() || res == flate2::Status::BufError {
+                        let bytes = this.buffer.remove_data(this.buffer.len()).to_vec();
+                        return Poll::Ready(Some(Ok(bytes.into())));
+                    }
+                }
+                EncoderState::Flushing => {
+                    let (_read, res) = this.encode(&[][..], FlushCompress::Finish)?;
+                    if !this.buffer.is_empty() {
+                        let bytes = this.buffer.remove_data(this.buffer.len()).to_vec();
+                        return Poll::Ready(Some(Ok(bytes.into())));
+                    }
+                    if res == flate2::Status::StreamEnd {
+                        this.state = EncoderState::Finished;
+                    }
+                }
+                EncoderState::Finished => return Poll::Ready(None),
+            }
+        }
+    }
+}

pbs-tools/src/fs.rs

@@ -1,13 +1,15 @@
 //! File system helper utilities.
 
 use std::borrow::{Borrow, BorrowMut};
+use std::collections::HashMap;
+use std::hash::BuildHasher;
 use std::ops::{Deref, DerefMut};
 use std::os::unix::io::{AsRawFd, RawFd};
 
 use anyhow::{bail, format_err, Error};
 use nix::dir;
 use nix::dir::Dir;
-use nix::fcntl::OFlag;
+use nix::fcntl::{AtFlags, OFlag};
 use nix::sys::stat::Mode;
 
 use regex::Regex;
@@ -344,3 +346,58 @@ fn do_lock_dir_noblock(
 
     Ok(handle)
 }
+
+pub fn complete_file_name<S>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String>
+where
+    S: BuildHasher,
+{
+    let mut result = vec![];
+
+    let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
+
+    let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
+        Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
+        Err(_) => false,
+    };
+
+    if !is_dir {
+        if let Some(parent) = dirname.parent() {
+            dirname = parent.to_owned();
+        }
+    }
+
+    let mut dir =
+        match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
+            Ok(d) => d,
+            Err(_) => return result,
+        };
+
+    for item in dir.iter() {
+        if let Ok(entry) = item {
+            if let Ok(name) = entry.file_name().to_str() {
+                if name == "." || name == ".." {
+                    continue;
+                }
+                let mut newpath = dirname.clone();
+                newpath.push(name);
+
+                if let Ok(stat) =
+                    nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
+                {
+                    if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
+                        newpath.push("");
+                        if let Some(newpath) = newpath.to_str() {
+                            result.push(newpath.to_owned());
+                        }
+                        continue;
+                    }
+                }
+                if let Some(newpath) = newpath.to_str() {
+                    result.push(newpath.to_owned());
+                }
+            }
+        }
+    }
+
+    result
+}

pbs-tools/src/lib.rs

@@ -1,11 +1,14 @@
+pub mod acl;
 pub mod auth;
 pub mod borrow;
 pub mod broadcast_future;
 pub mod cert;
+pub mod compression;
 pub mod format;
 pub mod fs;
 pub mod json;
 pub mod nom;
+pub mod ops;
 pub mod percent_encoding;
 pub mod process_locker;
 pub mod sha;
@@ -13,6 +16,8 @@ pub mod str;
 pub mod sync;
 pub mod ticket;
 pub mod tokio;
+pub mod xattr;
+pub mod zip;
 
 mod command;
 pub use command::{command_output, command_output_as_string, run_command};

pbs-tools/src/ops.rs

@@ -0,0 +1,12 @@
+//! std::ops extensions
+
+/// Modeled after the nightly `std::ops::ControlFlow`.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum ControlFlow<B, C = ()> {
+    Continue(C),
+    Break(B),
+}
+
+impl<B> ControlFlow<B> {
+    pub const CONTINUE: ControlFlow<B, ()> = ControlFlow::Continue(());
+}

pbs-tools/src/str.rs

@@ -15,3 +15,13 @@ pub fn join<S: Borrow<str>>(data: &[S], sep: char) -> String {
     list
 }
 
+pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
+    let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
+        Some(n) => &line[n..],
+        None => return &[],
+    };
+    match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
+        Some(n) => &line[..(line.len() - n)],
+        None => &[],
+    }
+}

pbs-tools/src/zip.rs

@@ -22,7 +22,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
 use crc32fast::Hasher;
 use proxmox::tools::time::gmtime;
 
-use crate::tools::compression::{DeflateEncoder, Level};
+use crate::compression::{DeflateEncoder, Level};
 
 const LOCAL_FH_SIG: u32 = 0x04034B50;
 const LOCAL_FF_SIG: u32 = 0x08074B50;

src/api2/access/user.rs

@@ -10,7 +10,11 @@ use proxmox::api::router::SubdirMap;
 use proxmox::api::schema::{Schema, StringSchema};
 use proxmox::tools::fs::open_file_locked;
 
-use crate::api2::types::*;
+use pbs_api_types::{
+    PASSWORD_FORMAT, PROXMOX_CONFIG_DIGEST_SCHEMA, SINGLE_LINE_COMMENT_SCHEMA, Authid,
+    Tokenname, UserWithTokens, Userid,
+};
+
 use crate::config::user;
 use crate::config::token_shadow;
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_PERMISSIONS_MODIFY};
@@ -22,68 +26,8 @@ pub const PBS_PASSWORD_SCHEMA: Schema = StringSchema::new("User Password.")
     .max_length(64)
     .schema();
 
-#[api(
+fn new_user_with_tokens(user: user::User) -> UserWithTokens {
-    properties: {
+    UserWithTokens {
-        userid: {
-            type: Userid,
-        },
-        comment: {
-            optional: true,
-            schema: SINGLE_LINE_COMMENT_SCHEMA,
-        },
-        enable: {
-            optional: true,
-            schema: user::ENABLE_USER_SCHEMA,
-        },
-        expire: {
-            optional: true,
-            schema: user::EXPIRE_USER_SCHEMA,
-        },
-        firstname: {
-            optional: true,
-            schema: user::FIRST_NAME_SCHEMA,
-        },
-        lastname: {
-            schema: user::LAST_NAME_SCHEMA,
-            optional: true,
-         },
-        email: {
-            schema: user::EMAIL_SCHEMA,
-            optional: true,
-        },
-        tokens: {
-            type: Array,
-            optional: true,
-            description: "List of user's API tokens.",
-            items: {
-                type: user::ApiToken
-            },
-        },
-    }
-)]
-#[derive(Serialize,Deserialize)]
-/// User properties with added list of ApiTokens
-pub struct UserWithTokens {
-    pub userid: Userid,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub comment: Option<String>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub enable: Option<bool>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub expire: Option<i64>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub firstname: Option<String>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub lastname: Option<String>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub email: Option<String>,
-    #[serde(skip_serializing_if="Vec::is_empty", default)]
-    pub tokens: Vec<user::ApiToken>,
-}
-
-impl UserWithTokens {
-    fn new(user: user::User) -> Self {
-        Self {
         userid: user.userid,
         comment: user.comment,
         enable: user.enable,
@@ -94,7 +38,6 @@ impl UserWithTokens {
         tokens: Vec::new(),
     }
 }
-}
 
 #[api(
     input: {
@@ -165,13 +108,13 @@ pub fn list_users(
             });
         iter
             .map(|user: user::User| {
-                let mut user = UserWithTokens::new(user);
+                let mut user = new_user_with_tokens(user);
                 user.tokens = user_to_tokens.remove(&user.userid).unwrap_or_default();
                 user
             })
             .collect()
     } else {
-        iter.map(UserWithTokens::new)
+        iter.map(new_user_with_tokens)
             .collect()
     };
 

src/api2/admin/datastore.rs

@@ -26,13 +26,14 @@ use proxmox::{http_err, identity, list_subdirs_api_method, sortable};
 use pxar::accessor::aio::Accessor;
 use pxar::EntryKind;
 
+use pbs_client::pxar::create_zip;
+
 use crate::api2::types::*;
 use crate::api2::node::rrd::create_value_from_rrd;
 use crate::api2::helpers;
 use crate::backup::*;
 use crate::config::datastore;
 use crate::config::cached_user_info::CachedUserInfo;
-use crate::pxar::create_zip;
 
 use crate::server::{jobstate::Job, WorkerTask};
 use crate::tools::{

src/api2/backup/mod.rs

@@ -13,6 +13,7 @@ use proxmox::api::router::SubdirMap;
 use proxmox::api::schema::*;
 
 use pbs_tools::fs::lock_dir_noblock_shared;
+use pbs_datastore::PROXMOX_BACKUP_PROTOCOL_ID_V1;
 
 use crate::tools;
 use crate::server::{WorkerTask, H2Service};

src/api2/config/remote.rs

@@ -6,8 +6,9 @@ use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
 use proxmox::http_err;
 use proxmox::tools::fs::open_file_locked;
 
+use pbs_client::{HttpClient, HttpClientOptions};
+
 use crate::api2::types::*;
-use crate::client::{HttpClient, HttpClientOptions};
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::remote;
 use crate::config::acl::{PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY};

src/api2/pull.rs

@@ -7,9 +7,10 @@ use futures::{select, future::FutureExt};
 use proxmox::api::api;
 use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
 
+use pbs_client::{HttpClient, BackupRepository};
+
 use crate::server::{WorkerTask, jobstate::Job, pull::pull_store};
 use crate::backup::DataStore;
-use crate::client::{HttpClient, BackupRepository};
 use crate::api2::types::{
     DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, Authid,
 };

src/api2/reader/mod.rs

@@ -28,6 +28,7 @@ use proxmox::{
 };
 
 use pbs_tools::fs::lock_dir_noblock_shared;
+use pbs_datastore::PROXMOX_BACKUP_READER_PROTOCOL_ID_V1;
 
 use crate::{
     api2::{

src/api2/types/mod.rs

@@ -107,14 +107,6 @@ pub const TAPE_ENCRYPTION_KEY_FINGERPRINT_SCHEMA: Schema = StringSchema::new(
     .format(&FINGERPRINT_SHA256_FORMAT)
     .schema();
 
-pub const PROXMOX_CONFIG_DIGEST_SCHEMA: Schema = StringSchema::new(
-    "Prevent changes if current configuration file has different \
-    SHA256 digest. This can be used to prevent concurrent \
-    modifications."
-)
-    .format(&PVE_CONFIG_DIGEST_FORMAT) .schema();
-
-
 pub const CHUNK_DIGEST_SCHEMA: Schema = StringSchema::new("Chunk digest (SHA256).")
     .format(&CHUNK_DIGEST_FORMAT)
     .schema();

src/backup/mod.rs

@@ -5,20 +5,6 @@ use anyhow::{bail, Error};
 // Note: .pcat1 => Proxmox Catalog Format version 1
 pub const CATALOG_NAME: &str = "catalog.pcat1.didx";
 
-#[macro_export]
-macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 {
-    () => {
-        "proxmox-backup-protocol-v1"
-    };
-}
-
-#[macro_export]
-macro_rules! PROXMOX_BACKUP_READER_PROTOCOL_ID_V1 {
-    () => {
-        "proxmox-backup-reader-protocol-v1"
-    };
-}
-
 /// Unix system user used by proxmox-backup-proxy
 pub const BACKUP_USER_NAME: &str = "backup";
 /// Unix system group used by proxmox-backup-proxy
@@ -102,9 +88,5 @@ pub use datastore::*;
 mod verify;
 pub use verify::*;
 
-// Move to client
-mod catalog_shell;
-pub use catalog_shell::*;
-
 mod cached_chunk_reader;
 pub use cached_chunk_reader::*;

src/bin/dump-catalog-shell-cli.rs

@@ -3,7 +3,7 @@ use anyhow::{Error};
 use proxmox::api::format::*;
 use proxmox::api::cli::*;
 
-use proxmox_backup::backup::catalog_shell_cli;
+use pbs_client::catalog_shell::catalog_shell_cli;
 
 fn main() -> Result<(), Error> {
 

src/bin/proxmox-backup-client.rs

@@ -27,47 +27,24 @@ use proxmox::{
 };
 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
 
-use pbs_datastore::catalog::BackupCatalogWriter;
+use pbs_api_types::CryptMode;
-use pbs_tools::sync::StdChannelWriter;
+use pbs_client::{
-use pbs_tools::tokio::TokioWriterAdapter;
+    BACKUP_SOURCE_SCHEMA,
-
+    BackupReader,
-use proxmox_backup::api2::types::*;
+    BackupRepository,
-use proxmox_backup::api2::version;
+    BackupSpecificationType,
-use proxmox_backup::client::*;
+    BackupStats,
-use proxmox_backup::backup::{
+    BackupWriter,
-    archive_type,
+    HttpClient,
-    decrypt_key,
+    PxarBackupStream,
-    rsa_encrypt_key_config,
+    RemoteChunkReader,
-    verify_chunk_size,
+    UploadOptions,
-    ArchiveType,
+    delete_ticket_info,
-    AsyncReadChunk,
+    parse_backup_specification,
-    BackupDir,
+    view_task_result,
-    BackupGroup,
-    BackupManifest,
-    BufferedDynamicReader,
-    CATALOG_NAME,
-    CatalogReader,
-    CatalogWriter,
-    ChunkStream,
-    CryptConfig,
-    CryptMode,
-    DynamicIndexReader,
-    ENCRYPTED_KEY_BLOB_NAME,
-    FixedChunkStream,
-    FixedIndexReader,
-    KeyConfig,
-    IndexFile,
-    MANIFEST_BLOB_NAME,
-    Shell,
-    PruneOptions,
 };
-use proxmox_backup::tools;
+use pbs_client::catalog_shell::Shell;
-
+use pbs_client::tools::{
-mod proxmox_backup_client;
-use proxmox_backup_client::*;
-
-pub mod proxmox_client_tools;
-use proxmox_client_tools::{
     complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot,
     complete_backup_source, complete_chunk_size, complete_group_or_snapshot,
     complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect,
@@ -78,6 +55,37 @@ use proxmox_client_tools::{
     },
     CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA,
 };
+use pbs_datastore::CryptConfig;
+use pbs_datastore::backup_info::{BackupDir, BackupGroup};
+use pbs_datastore::catalog::BackupCatalogWriter;
+use pbs_datastore::dynamic_index::DynamicIndexReader;
+use pbs_datastore::fixed_index::FixedIndexReader;
+use pbs_datastore::index::IndexFile;
+use pbs_datastore::manifest::{MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, archive_type};
+use pbs_datastore::read_chunk::AsyncReadChunk;
+use pbs_tools::sync::StdChannelWriter;
+use pbs_tools::tokio::TokioWriterAdapter;
+
+use proxmox_backup::api2::types::*;
+use proxmox_backup::api2::version;
+use proxmox_backup::backup::{
+    decrypt_key,
+    rsa_encrypt_key_config,
+    verify_chunk_size,
+    BufferedDynamicReader,
+    CATALOG_NAME,
+    CatalogReader,
+    CatalogWriter,
+    ChunkStream,
+    ENCRYPTED_KEY_BLOB_NAME,
+    FixedChunkStream,
+    KeyConfig,
+    PruneOptions,
+};
+use proxmox_backup::tools;
+
+mod proxmox_backup_client;
+use proxmox_backup_client::*;
 
 fn record_repository(repo: &BackupRepository) {
 
@@ -172,7 +180,7 @@ async fn backup_directory<P: AsRef<Path>>(
     archive_name: &str,
     chunk_size: Option<usize>,
     catalog: Arc<Mutex<CatalogWriter<TokioWriterAdapter<StdChannelWriter>>>>,
-    pxar_create_options: proxmox_backup::pxar::PxarCreateOptions,
+    pxar_create_options: pbs_client::pxar::PxarCreateOptions,
     upload_options: UploadOptions,
 ) -> Result<BackupStats, Error> {
 
@@ -589,7 +597,7 @@ fn spawn_catalog_upload(
                type: Integer,
                description: "Max number of entries to hold in memory.",
                optional: true,
-               default: proxmox_backup::pxar::ENCODER_MAX_ENTRIES as isize,
+               default: pbs_client::pxar::ENCODER_MAX_ENTRIES as isize,
            },
            "verbose": {
                type: Boolean,
@@ -633,7 +641,7 @@ async fn create_backup(
     let include_dev = param["include-dev"].as_array();
 
     let entries_max = param["entries-max"].as_u64()
-        .unwrap_or(proxmox_backup::pxar::ENCODER_MAX_ENTRIES as u64);
+        .unwrap_or(pbs_client::pxar::ENCODER_MAX_ENTRIES as u64);
 
     let empty = Vec::new();
     let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
@@ -856,7 +864,7 @@ async fn create_backup(
                 println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
                 catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
 
-                let pxar_options = proxmox_backup::pxar::PxarCreateOptions {
+                let pxar_options = pbs_client::pxar::PxarCreateOptions {
                     device_set: devices.clone(),
                     patterns: pattern_list.clone(),
                     entries_max: entries_max as usize,
@@ -1168,7 +1176,7 @@ async fn restore(param: Value) -> Result<Value, Error> {
 
         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
 
-        let options = proxmox_backup::pxar::PxarExtractOptions {
+        let options = pbs_client::pxar::PxarExtractOptions {
             match_list: &[],
             extract_match_default: true,
             allow_existing_dirs,
@@ -1176,10 +1184,10 @@ async fn restore(param: Value) -> Result<Value, Error> {
         };
 
         if let Some(target) = target {
-            proxmox_backup::pxar::extract_archive(
+            pbs_client::pxar::extract_archive(
                 pxar::decoder::Decoder::from_std(reader)?,
                 Path::new(target),
-                proxmox_backup::pxar::Flags::DEFAULT,
+                pbs_client::pxar::Flags::DEFAULT,
                 |path| {
                     if verbose {
                         println!("{:?}", path);
@@ -1377,7 +1385,6 @@ async fn status(param: Value) -> Result<Value, Error> {
     Ok(Value::Null)
 }
 
-use proxmox_backup::client::RemoteChunkReader;
 /// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
 /// async use!
 ///
@@ -1424,13 +1431,13 @@ fn main() {
         .arg_param(&["backupspec"])
         .completion_cb("repository", complete_repository)
         .completion_cb("backupspec", complete_backup_source)
-        .completion_cb("keyfile", tools::complete_file_name)
+        .completion_cb("keyfile", pbs_tools::fs::complete_file_name)
-        .completion_cb("master-pubkey-file", tools::complete_file_name)
+        .completion_cb("master-pubkey-file", pbs_tools::fs::complete_file_name)
         .completion_cb("chunk-size", complete_chunk_size);
 
     let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK)
         .completion_cb("repository", complete_repository)
-        .completion_cb("keyfile", tools::complete_file_name);
+        .completion_cb("keyfile", pbs_tools::fs::complete_file_name);
 
     let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS)
         .completion_cb("repository", complete_repository);
@@ -1443,7 +1450,7 @@ fn main() {
         .completion_cb("repository", complete_repository)
         .completion_cb("snapshot", complete_group_or_snapshot)
         .completion_cb("archive-name", complete_archive_name)
-        .completion_cb("target", tools::complete_file_name);
+        .completion_cb("target", pbs_tools::fs::complete_file_name);
 
     let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE)
         .arg_param(&["group"])

src/bin/proxmox-backup-manager.rs

@@ -6,12 +6,12 @@ use serde_json::{json, Value};
 
 use proxmox::api::{api, cli::*, RpcEnvironment};
 
+use pbs_client::{connect_to_localhost, display_task_log, view_task_result};
 use pbs_tools::percent_encoding::percent_encode_component;
 
 use proxmox_backup::tools;
 use proxmox_backup::config;
 use proxmox_backup::api2::{self, types::* };
-use proxmox_backup::client::*;
 use proxmox_backup::server::wait_for_local_worker;
 
 mod proxmox_backup_manager;

src/bin/proxmox-file-restore.rs

@@ -16,18 +16,10 @@ use proxmox::api::{
 use pxar::accessor::aio::Accessor;
 use pxar::decoder::aio::Decoder;
 
-use proxmox_backup::api2::{helpers, types::ArchiveEntry};
+use pbs_datastore::index::IndexFile;
-use proxmox_backup::backup::{
+use pbs_client::{BackupReader, RemoteChunkReader};
-    decrypt_key, BackupDir, BufferedDynamicReader, CatalogReader, CryptConfig, CryptMode,
+use pbs_client::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq};
-    DirEntryAttribute, IndexFile, LocalDynamicReadAt, CATALOG_NAME,
+use pbs_client::tools::{
-};
-use proxmox_backup::client::{BackupReader, RemoteChunkReader};
-use proxmox_backup::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq};
-use proxmox_backup::tools;
-
-// use "pub" so rust doesn't complain about "unused" functions in the module
-pub mod proxmox_client_tools;
-use proxmox_client_tools::{
     complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value,
     key_source::{
         crypto_parameters_keep_fd, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
@@ -36,6 +28,13 @@ use proxmox_client_tools::{
     REPO_URL_SCHEMA,
 };
 
+use proxmox_backup::api2::{helpers, types::ArchiveEntry};
+use proxmox_backup::backup::{
+    decrypt_key, BackupDir, BufferedDynamicReader, CatalogReader, CryptConfig, CryptMode,
+    DirEntryAttribute, LocalDynamicReadAt, CATALOG_NAME,
+};
+use proxmox_backup::tools;
+
 mod proxmox_file_restore;
 use proxmox_file_restore::*;
 
@@ -456,7 +455,7 @@ fn main() {
         .arg_param(&["snapshot", "path", "target"])
         .completion_cb("repository", complete_repository)
         .completion_cb("snapshot", complete_group_or_snapshot)
-        .completion_cb("target", tools::complete_file_name);
+        .completion_cb("target", pbs_tools::fs::complete_file_name);
 
     let status_cmd_def = CliCommand::new(&API_METHOD_STATUS);
     let stop_cmd_def = CliCommand::new(&API_METHOD_STOP)
@@ -476,3 +475,15 @@ fn main() {
         Some(|future| pbs_runtime::main(future)),
     );
 }
+
+/// Returns a runtime dir owned by the current user.
+/// Note that XDG_RUNTIME_DIR is not always available, especially for non-login users like
+/// "www-data", so we use a custom one in /run/proxmox-backup/<uid> instead.
+pub fn get_user_run_dir() -> Result<std::path::PathBuf, Error> {
+    let uid = nix::unistd::Uid::current();
+    let mut path: std::path::PathBuf = pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR.into();
+    path.push(uid.to_string());
+    tools::create_run_dir()?;
+    std::fs::create_dir_all(&path)?;
+    Ok(path)
+}

src/bin/proxmox-restore-daemon.rs

@@ -13,8 +13,9 @@ use std::sync::{Arc, Mutex};
 use tokio::sync::mpsc;
 use tokio_stream::wrappers::ReceiverStream;
 
+use pbs_client::DEFAULT_VSOCK_PORT;
+
 use proxmox::api::RpcEnvironmentType;
-use proxmox_backup::client::DEFAULT_VSOCK_PORT;
 use proxmox_backup::server::{rest::*, ApiConfig};
 
 mod proxmox_restore_daemon;

src/bin/proxmox-tape.rs

@@ -14,6 +14,7 @@ use proxmox::{
     },
 };
 
+use pbs_client::{connect_to_localhost, view_task_result};
 use pbs_tools::format::{
     HumanByte,
     render_epoch,
@@ -21,10 +22,6 @@ use pbs_tools::format::{
 };
 
 use proxmox_backup::{
-    client::{
-        connect_to_localhost,
-        view_task_result,
-    },
     api2::{
         self,
         types::{

src/bin/proxmox_backup_client/benchmark.rs

@@ -18,6 +18,9 @@ use proxmox::api::{
     router::ReturnType,
 };
 
+use pbs_client::tools::key_source::get_encryption_key_password;
+use pbs_client::{BackupRepository, BackupWriter};
+
 use proxmox_backup::backup::{
     load_and_decrypt_key,
     CryptConfig,
@@ -25,8 +28,6 @@ use proxmox_backup::backup::{
     DataChunkBuilder,
 };
 
-use proxmox_backup::client::*;
-
 use crate::{
     KEYFILE_SCHEMA, REPO_URL_SCHEMA,
     extract_repository_from_value,
@@ -34,8 +35,6 @@ use crate::{
     connect,
 };
 
-use crate::proxmox_client_tools::key_source::get_encryption_key_password;
-
 #[api()]
 #[derive(Copy, Clone, Serialize)]
 /// Speed test result

src/bin/proxmox_backup_client/catalog.rs

@@ -7,9 +7,10 @@ use serde_json::Value;
 
 use proxmox::api::{api, cli::*};
 
-use proxmox_backup::tools;
+use pbs_client::tools::key_source::get_encryption_key_password;
+use pbs_client::{BackupReader, RemoteChunkReader};
 
-use proxmox_backup::client::*;
+use proxmox_backup::tools;
 
 use crate::{
     REPO_URL_SCHEMA,
@@ -37,8 +38,6 @@ use crate::{
     Shell,
 };
 
-use crate::proxmox_client_tools::key_source::get_encryption_key_password;
-
 #[api(
    input: {
         properties: {
@@ -219,9 +218,9 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), file_info.chunk_crypt_mode(), most_used);
     let reader = BufferedDynamicReader::new(index, chunk_reader);
     let archive_size = reader.archive_size();
-    let reader: proxmox_backup::pxar::fuse::Reader =
+    let reader: pbs_client::pxar::fuse::Reader =
         Arc::new(BufferedDynamicReadAt::new(reader));
-    let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
+    let decoder = pbs_client::pxar::fuse::Accessor::new(reader, archive_size).await?;
 
     client.download(CATALOG_NAME, &mut tmpfile).await?;
     let index = DynamicIndexReader::new(tmpfile)

src/bin/proxmox_backup_client/key.rs

@@ -14,19 +14,18 @@ use proxmox::sys::linux::tty;
 use proxmox::tools::fs::{file_get_contents, replace_file, CreateOptions};
 
 use pbs_datastore::{KeyInfo, Kdf};
+use pbs_client::tools::key_source::{
+    find_default_encryption_key, find_default_master_pubkey, get_encryption_key_password,
+    place_default_encryption_key, place_default_master_pubkey,
+};
+
 
 use proxmox_backup::{
     api2::types::{RsaPubKeyInfo, PASSWORD_HINT_SCHEMA},
     backup::{rsa_decrypt_key_config, KeyConfig},
-    tools,
     tools::paperkey::{generate_paper_key, PaperkeyFormat},
 };
 
-use crate::proxmox_client_tools::key_source::{
-    find_default_encryption_key, find_default_master_pubkey, get_encryption_key_password,
-    place_default_encryption_key, place_default_master_pubkey,
-};
-
 #[api(
     input: {
         properties: {
@@ -458,35 +457,35 @@ fn paper_key(
 pub fn cli() -> CliCommandMap {
     let key_create_cmd_def = CliCommand::new(&API_METHOD_CREATE)
         .arg_param(&["path"])
-        .completion_cb("path", tools::complete_file_name);
+        .completion_cb("path", pbs_tools::fs::complete_file_name);
 
     let key_import_with_master_key_cmd_def = CliCommand::new(&API_METHOD_IMPORT_WITH_MASTER_KEY)
         .arg_param(&["master-keyfile"])
-        .completion_cb("master-keyfile", tools::complete_file_name)
+        .completion_cb("master-keyfile", pbs_tools::fs::complete_file_name)
         .arg_param(&["encrypted-keyfile"])
-        .completion_cb("encrypted-keyfile", tools::complete_file_name)
+        .completion_cb("encrypted-keyfile", pbs_tools::fs::complete_file_name)
         .arg_param(&["path"])
-        .completion_cb("path", tools::complete_file_name);
+        .completion_cb("path", pbs_tools::fs::complete_file_name);
 
     let key_change_passphrase_cmd_def = CliCommand::new(&API_METHOD_CHANGE_PASSPHRASE)
         .arg_param(&["path"])
-        .completion_cb("path", tools::complete_file_name);
+        .completion_cb("path", pbs_tools::fs::complete_file_name);
 
     let key_create_master_key_cmd_def = CliCommand::new(&API_METHOD_CREATE_MASTER_KEY);
     let key_import_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_IMPORT_MASTER_PUBKEY)
         .arg_param(&["path"])
-        .completion_cb("path", tools::complete_file_name);
+        .completion_cb("path", pbs_tools::fs::complete_file_name);
     let key_show_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_SHOW_MASTER_PUBKEY)
         .arg_param(&["path"])
-        .completion_cb("path", tools::complete_file_name);
+        .completion_cb("path", pbs_tools::fs::complete_file_name);
 
     let key_show_cmd_def = CliCommand::new(&API_METHOD_SHOW_KEY)
         .arg_param(&["path"])
-        .completion_cb("path", tools::complete_file_name);
+        .completion_cb("path", pbs_tools::fs::complete_file_name);
 
     let paper_key_cmd_def = CliCommand::new(&API_METHOD_PAPER_KEY)
         .arg_param(&["path"])
-        .completion_cb("path", tools::complete_file_name);
+        .completion_cb("path", pbs_tools::fs::complete_file_name);
 
     CliCommandMap::new()
         .insert("create", key_create_cmd_def)

src/bin/proxmox_backup_client/mount.rs

@@ -17,6 +17,9 @@ use proxmox::{sortable, identity};
 use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment, schema::*, cli::*};
 use proxmox::tools::fd::Fd;
 
+use pbs_client::tools::key_source::get_encryption_key_password;
+use pbs_client::{BackupReader, RemoteChunkReader};
+
 use proxmox_backup::tools;
 use proxmox_backup::backup::{
     load_and_decrypt_key,
@@ -28,8 +31,6 @@ use proxmox_backup::backup::{
     CachedChunkReader,
 };
 
-use proxmox_backup::client::*;
-
 use crate::{
     REPO_URL_SCHEMA,
     extract_repository_from_value,
@@ -43,8 +44,6 @@ use crate::{
     BufferedDynamicReadAt,
 };
 
-use crate::proxmox_client_tools::key_source::get_encryption_key_password;
-
 #[sortable]
 const API_METHOD_MOUNT: ApiMethod = ApiMethod::new(
     &ApiHandler::Sync(&mount),
@@ -98,7 +97,7 @@ pub fn mount_cmd_def() -> CliCommand {
         .completion_cb("repository", complete_repository)
         .completion_cb("snapshot", complete_group_or_snapshot)
         .completion_cb("archive-name", complete_pxar_archive_name)
-        .completion_cb("target", tools::complete_file_name)
+        .completion_cb("target", pbs_tools::fs::complete_file_name)
 }
 
 pub fn map_cmd_def() -> CliCommand {
@@ -257,11 +256,11 @@ async fn mount_do(param: Value, pipe: Option<Fd>) -> Result<Value, Error> {
         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
         let reader = BufferedDynamicReader::new(index, chunk_reader);
         let archive_size = reader.archive_size();
-        let reader: proxmox_backup::pxar::fuse::Reader =
+        let reader: pbs_client::pxar::fuse::Reader =
             Arc::new(BufferedDynamicReadAt::new(reader));
-        let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
+        let decoder = pbs_client::pxar::fuse::Accessor::new(reader, archive_size).await?;
 
-        let session = proxmox_backup::pxar::fuse::Session::mount(
+        let session = pbs_client::pxar::fuse::Session::mount(
             decoder,
             &options,
             false,

src/bin/proxmox_backup_client/snapshot.rs

@@ -8,6 +8,8 @@ use proxmox::{
     tools::fs::file_get_contents,
 };
 
+use pbs_client::tools::key_source::get_encryption_key_password;
+
 use proxmox_backup::{
     tools,
     api2::types::*,
@@ -35,8 +37,6 @@ use crate::{
     record_repository,
 };
 
-use crate::proxmox_client_tools::key_source::get_encryption_key_password;
-
 #[api(
    input: {
         properties: {
@@ -412,8 +412,8 @@ pub fn snapshot_mgtm_cli() -> CliCommandMap {
             CliCommand::new(&API_METHOD_UPLOAD_LOG)
                 .arg_param(&["snapshot", "logfile"])
                 .completion_cb("snapshot", complete_backup_snapshot)
-                .completion_cb("logfile", tools::complete_file_name)
+                .completion_cb("logfile", pbs_tools::fs::complete_file_name)
-                .completion_cb("keyfile", tools::complete_file_name)
+                .completion_cb("keyfile", pbs_tools::fs::complete_file_name)
                 .completion_cb("repository", complete_repository)
         )
 }

src/bin/proxmox_backup_client/task.rs

@@ -4,10 +4,10 @@ use serde_json::{json, Value};
 use proxmox::api::{api, cli::*};
 
 use pbs_tools::percent_encoding::percent_encode_component;
+use pbs_client::display_task_log;
 
 use proxmox_backup::tools;
 
-use proxmox_backup::client::*;
 use proxmox_backup::api2::types::UPID_SCHEMA;
 
 use crate::{

src/bin/proxmox_backup_manager/datastore.rs

@@ -3,12 +3,10 @@ use serde_json::Value;
 
 use proxmox::api::{api, cli::*, RpcEnvironment, ApiHandler};
 
+use pbs_client::{connect_to_localhost, view_task_result};
+
 use proxmox_backup::config;
 use proxmox_backup::api2::{self, types::* };
-use proxmox_backup::client::{
-    connect_to_localhost,
-    view_task_result,
-};
 use proxmox_backup::config::datastore::DIR_NAME_SCHEMA;
 
 #[api(

src/bin/proxmox_file_restore/block_driver.rs

@@ -1,19 +1,20 @@
 //! Abstraction layer over different methods of accessing a block backup
-use anyhow::{bail, Error};
-use serde::{Deserialize, Serialize};
-use serde_json::{json, Value};
-
 use std::collections::HashMap;
 use std::future::Future;
 use std::hash::BuildHasher;
 use std::pin::Pin;
 
-use proxmox_backup::backup::{BackupDir, BackupManifest};
+use anyhow::{bail, Error};
-use proxmox_backup::api2::types::ArchiveEntry;
+use serde::{Deserialize, Serialize};
-use proxmox_backup::client::BackupRepository;
+use serde_json::{json, Value};
 
 use proxmox::api::{api, cli::*};
 
+use pbs_client::BackupRepository;
+
+use proxmox_backup::backup::{BackupDir, BackupManifest};
+use proxmox_backup::api2::types::ArchiveEntry;
+
 use super::block_driver_qemu::QemuBlockDriver;
 
 /// Contains details about a snapshot that is to be accessed by block file restore

src/bin/proxmox_file_restore/block_driver_qemu.rs

@@ -1,21 +1,23 @@
 //! Block file access via a small QEMU restore VM using the PBS block driver in QEMU
+use std::collections::HashMap;
+use std::fs::{File, OpenOptions};
+use std::io::{prelude::*, SeekFrom};
+
 use anyhow::{bail, Error};
 use futures::FutureExt;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 
-use std::collections::HashMap;
-use std::fs::{File, OpenOptions};
-use std::io::{prelude::*, SeekFrom};
-
 use proxmox::tools::fs::lock_file;
+
+use pbs_client::{DEFAULT_VSOCK_PORT, BackupRepository, VsockClient};
+
 use proxmox_backup::api2::types::ArchiveEntry;
 use proxmox_backup::backup::BackupDir;
-use proxmox_backup::client::*;
 use proxmox_backup::tools;
 
 use super::block_driver::*;
-use crate::proxmox_client_tools::get_user_run_dir;
+use crate::get_user_run_dir;
 
 const RESTORE_VM_MAP: &str = "restore-vm-map.json";
 

src/bin/proxmox_file_restore/qemu_helper.rs

@@ -13,8 +13,9 @@ use nix::unistd::Pid;
 
 use proxmox::tools::fs::{create_path, file_read_string, make_tmp_file, CreateOptions};
 
+use pbs_client::{VsockClient, DEFAULT_VSOCK_PORT};
+
 use proxmox_backup::backup::backup_user;
-use proxmox_backup::client::{VsockClient, DEFAULT_VSOCK_PORT};
 use proxmox_backup::tools;
 
 use super::SnapRestoreDetails;

src/bin/proxmox_restore_daemon/api.rs

@@ -19,12 +19,13 @@ use proxmox::api::{
 };
 use proxmox::{identity, list_subdirs_api_method, sortable};
 
+use pbs_client::pxar::{create_archive, Flags, PxarCreateOptions, ENCODER_MAX_ENTRIES};
 use pbs_tools::fs::read_subdir;
+use pbs_tools::zip::zip_directory;
 
 use proxmox_backup::api2::types::*;
 use proxmox_backup::backup::DirEntryAttribute;
-use proxmox_backup::pxar::{create_archive, Flags, PxarCreateOptions, ENCODER_MAX_ENTRIES};
+use proxmox_backup::tools;
-use proxmox_backup::tools::{self, zip::zip_directory};
 
 use pxar::encoder::aio::TokioWriter;
 

src/bin/proxmox_tape/backup_job.rs

@@ -3,12 +3,10 @@ use serde_json::Value;
 
 use proxmox::api::{api, cli::*, RpcEnvironment, ApiHandler};
 
+use pbs_client::{connect_to_localhost, view_task_result};
+
 use proxmox_backup::{
     config,
-    client::{
-        connect_to_localhost,
-        view_task_result,
-    },
     api2::{
         self,
         types::*,

src/bin/pxar.rs

@@ -12,13 +12,11 @@ use futures::select;
 use tokio::signal::unix::{signal, SignalKind};
 
 use pathpatterns::{MatchEntry, MatchType, PatternFlag};
+use pbs_client::pxar::{fuse, format_single_line_entry, ENCODER_MAX_ENTRIES, Flags, PxarExtractOptions};
 
 use proxmox::api::cli::*;
 use proxmox::api::api;
 
-use proxmox_backup::tools;
-use proxmox_backup::pxar::{fuse, format_single_line_entry, ENCODER_MAX_ENTRIES, Flags, PxarExtractOptions};
-
 fn extract_archive_from_reader<R: std::io::Read>(
     reader: &mut R,
     target: &str,
@@ -26,8 +24,7 @@ fn extract_archive_from_reader<R: std::io::Read>(
     verbose: bool,
     options: PxarExtractOptions,
 ) -> Result<(), Error> {
-
+    pbs_client::pxar::extract_archive(
-    proxmox_backup::pxar::extract_archive(
         pxar::decoder::Decoder::from_std(reader)?,
         Path::new(target),
         feature_flags,
@@ -327,7 +324,7 @@ async fn create_archive(
         Some(HashSet::new())
     };
 
-    let options = proxmox_backup::pxar::PxarCreateOptions {
+    let options = pbs_client::pxar::PxarCreateOptions {
         entries_max: entries_max as usize,
         device_set,
         patterns,
@@ -372,7 +369,7 @@ async fn create_archive(
     }
 
     let writer = pxar::encoder::sync::StandardWriter::new(writer);
-    proxmox_backup::pxar::create_archive(
+    pbs_client::pxar::create_archive(
         dir,
         writer,
         feature_flags,
@@ -464,29 +461,29 @@ fn main() {
             "create",
             CliCommand::new(&API_METHOD_CREATE_ARCHIVE)
                 .arg_param(&["archive", "source"])
-                .completion_cb("archive", tools::complete_file_name)
+                .completion_cb("archive", pbs_tools::fs::complete_file_name)
-                .completion_cb("source", tools::complete_file_name),
+                .completion_cb("source", pbs_tools::fs::complete_file_name),
         )
         .insert(
             "extract",
             CliCommand::new(&API_METHOD_EXTRACT_ARCHIVE)
                 .arg_param(&["archive", "target"])
-                .completion_cb("archive", tools::complete_file_name)
+                .completion_cb("archive", pbs_tools::fs::complete_file_name)
-                .completion_cb("target", tools::complete_file_name)
+                .completion_cb("target", pbs_tools::fs::complete_file_name)
-                .completion_cb("files-from", tools::complete_file_name),
+                .completion_cb("files-from", pbs_tools::fs::complete_file_name),
         )
         .insert(
             "mount",
             CliCommand::new(&API_METHOD_MOUNT_ARCHIVE)
                 .arg_param(&["archive", "mountpoint"])
-                .completion_cb("archive", tools::complete_file_name)
+                .completion_cb("archive", pbs_tools::fs::complete_file_name)
-                .completion_cb("mountpoint", tools::complete_file_name),
+                .completion_cb("mountpoint", pbs_tools::fs::complete_file_name),
         )
         .insert(
             "list",
             CliCommand::new(&API_METHOD_DUMP_ARCHIVE)
                 .arg_param(&["archive"])
-                .completion_cb("archive", tools::complete_file_name),
+                .completion_cb("archive", pbs_tools::fs::complete_file_name),
         );
 
     let rpcenv = CliEnvironment::new();

src/config/user.rs

@@ -3,10 +3,8 @@ use std::sync::{Arc, RwLock};
 
 use anyhow::{bail, Error};
 use lazy_static::lazy_static;
-use serde::{Serialize, Deserialize};
 
 use proxmox::api::{
-    api,
     schema::*,
     section_config::{
         SectionConfig,
@@ -17,154 +15,18 @@ use proxmox::api::{
 
 use proxmox::tools::{fs::replace_file, fs::CreateOptions};
 
-use crate::api2::types::*;
+use pbs_api_types::{Authid, Userid};
+pub use pbs_api_types::{ApiToken, User};
+pub use pbs_api_types::{
+    EMAIL_SCHEMA, ENABLE_USER_SCHEMA, EXPIRE_USER_SCHEMA, FIRST_NAME_SCHEMA, LAST_NAME_SCHEMA,
+};
+
 use crate::tools::Memcom;
 
 lazy_static! {
     pub static ref CONFIG: SectionConfig = init();
 }
 
-pub const ENABLE_USER_SCHEMA: Schema = BooleanSchema::new(
-    "Enable the account (default). You can set this to '0' to disable the account.")
-    .default(true)
-    .schema();
-
-pub const EXPIRE_USER_SCHEMA: Schema = IntegerSchema::new(
-    "Account expiration date (seconds since epoch). '0' means no expiration date.")
-    .default(0)
-    .minimum(0)
-    .schema();
-
-pub const FIRST_NAME_SCHEMA: Schema = StringSchema::new("First name.")
-    .format(&SINGLE_LINE_COMMENT_FORMAT)
-    .min_length(2)
-    .max_length(64)
-    .schema();
-
-pub const LAST_NAME_SCHEMA: Schema = StringSchema::new("Last name.")
-    .format(&SINGLE_LINE_COMMENT_FORMAT)
-    .min_length(2)
-    .max_length(64)
-    .schema();
-
-pub const EMAIL_SCHEMA: Schema = StringSchema::new("E-Mail Address.")
-    .format(&SINGLE_LINE_COMMENT_FORMAT)
-    .min_length(2)
-    .max_length(64)
-    .schema();
-
-#[api(
-    properties: {
-        tokenid: {
-            schema: PROXMOX_TOKEN_ID_SCHEMA,
-        },
-        comment: {
-            optional: true,
-            schema: SINGLE_LINE_COMMENT_SCHEMA,
-        },
-        enable: {
-            optional: true,
-            schema: ENABLE_USER_SCHEMA,
-        },
-        expire: {
-            optional: true,
-            schema: EXPIRE_USER_SCHEMA,
-        },
-    }
-)]
-#[derive(Serialize,Deserialize)]
-/// ApiToken properties.
-pub struct ApiToken {
-    pub tokenid: Authid,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub comment: Option<String>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub enable: Option<bool>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub expire: Option<i64>,
-}
-
-impl ApiToken {
-
-    pub fn is_active(&self) -> bool {
-        if !self.enable.unwrap_or(true) {
-            return false;
-        }
-        if let Some(expire) = self.expire {
-            let now =  proxmox::tools::time::epoch_i64();
-            if expire > 0 && expire <= now {
-                return false;
-            }
-        }
-        true
-    }
-}
-
-#[api(
-    properties: {
-        userid: {
-            type: Userid,
-        },
-        comment: {
-            optional: true,
-            schema: SINGLE_LINE_COMMENT_SCHEMA,
-        },
-        enable: {
-            optional: true,
-            schema: ENABLE_USER_SCHEMA,
-        },
-        expire: {
-            optional: true,
-            schema: EXPIRE_USER_SCHEMA,
-        },
-        firstname: {
-            optional: true,
-            schema: FIRST_NAME_SCHEMA,
-        },
-        lastname: {
-            schema: LAST_NAME_SCHEMA,
-            optional: true,
-         },
-        email: {
-            schema: EMAIL_SCHEMA,
-            optional: true,
-        },
-    }
-)]
-#[derive(Serialize,Deserialize)]
-/// User properties.
-pub struct User {
-    pub userid: Userid,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub comment: Option<String>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub enable: Option<bool>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub expire: Option<i64>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub firstname: Option<String>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub lastname: Option<String>,
-    #[serde(skip_serializing_if="Option::is_none")]
-    pub email: Option<String>,
-}
-
-impl User {
-
-    pub fn is_active(&self) -> bool {
-        if !self.enable.unwrap_or(true) {
-            return false;
-        }
-        if let Some(expire) = self.expire {
-            let now =  proxmox::tools::time::epoch_i64();
-            if expire > 0 && expire <= now {
-                return false;
-            }
-        }
-        true
-    }
-}
-
 fn init() -> SectionConfig {
     let mut config = SectionConfig::new(&Authid::API_SCHEMA);
 

src/lib.rs

@@ -9,8 +9,6 @@ pub mod tools;
 #[macro_use]
 pub mod server;
 
-pub mod pxar;
-
 #[macro_use]
 pub mod backup;
 
@@ -18,8 +16,6 @@ pub mod config;
 
 pub mod api2;
 
-pub mod client;
-
 pub mod auth_helpers;
 
 pub mod auth;

src/server/pull.rs

@@ -22,10 +22,10 @@ use pbs_datastore::manifest::{
     CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, FileInfo, archive_type
 };
 use pbs_tools::sha::sha256;
+use pbs_client::{BackupReader, BackupRepository, HttpClient, HttpClientOptions, RemoteChunkReader};
 
 use crate::{
     backup::DataStore,
-    client::{BackupReader, BackupRepository, HttpClient, HttpClientOptions, RemoteChunkReader},
     server::WorkerTask,
     tools::ParallelHandler,
 };

src/server/rest.rs

@@ -30,6 +30,8 @@ use proxmox::api::{
 };
 use proxmox::http_err;
 
+use pbs_tools::compression::{DeflateEncoder, Level};
+
 use super::auth::AuthError;
 use super::environment::RestEnvironment;
 use super::formatter::*;
@@ -39,7 +41,7 @@ use crate::api2::types::{Authid, Userid};
 use crate::auth_helpers::*;
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::tools;
-use crate::tools::compression::{CompressionMethod, DeflateEncoder, Level};
+use crate::tools::compression::CompressionMethod;
 use crate::tools::AsyncReaderStream;
 use crate::tools::FileLogger;
 

src/tools/compression.rs

@@ -1,19 +1,5 @@
-use std::io;
-use std::pin::Pin;
-use std::task::{Context, Poll};
-
 use anyhow::{bail, Error};
-use bytes::Bytes;
-use flate2::{Compress, Compression, FlushCompress};
-use futures::ready;
-use futures::stream::Stream;
 use hyper::header;
-use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
-
-use proxmox::io_format_err;
-use proxmox::tools::byte_buffer::ByteBuffer;
-
-const BUFFER_SIZE: usize = 8192;
 
 /// Possible Compression Methods, order determines preference (later is preferred)
 #[derive(Eq, Ord, PartialEq, PartialOrd, Debug)]
@@ -51,182 +37,3 @@ impl std::str::FromStr for CompressionMethod {
         }
     }
 }
-
-pub enum Level {
-    Fastest,
-    Best,
-    Default,
-    Precise(u32),
-}
-
-#[derive(Eq, PartialEq)]
-enum EncoderState {
-    Reading,
-    Writing,
-    Flushing,
-    Finished,
-}
-
-pub struct DeflateEncoder<T> {
-    inner: T,
-    compressor: Compress,
-    buffer: ByteBuffer,
-    input_buffer: Bytes,
-    state: EncoderState,
-}
-
-impl<T> DeflateEncoder<T> {
-    pub fn new(inner: T) -> Self {
-        Self::with_quality(inner, Level::Default)
-    }
-
-    pub fn with_quality(inner: T, level: Level) -> Self {
-        let level = match level {
-            Level::Fastest => Compression::fast(),
-            Level::Best => Compression::best(),
-            Level::Default => Compression::new(3),
-            Level::Precise(val) => Compression::new(val),
-        };
-
-        Self {
-            inner,
-            compressor: Compress::new(level, false),
-            buffer: ByteBuffer::with_capacity(BUFFER_SIZE),
-            input_buffer: Bytes::new(),
-            state: EncoderState::Reading,
-        }
-    }
-
-    pub fn total_in(&self) -> u64 {
-        self.compressor.total_in()
-    }
-
-    pub fn total_out(&self) -> u64 {
-        self.compressor.total_out()
-    }
-
-    pub fn into_inner(self) -> T {
-        self.inner
-    }
-
-    fn encode(
-        &mut self,
-        inbuf: &[u8],
-        flush: FlushCompress,
-    ) -> Result<(usize, flate2::Status), io::Error> {
-        let old_in = self.compressor.total_in();
-        let old_out = self.compressor.total_out();
-        let res = self
-            .compressor
-            .compress(&inbuf[..], self.buffer.get_free_mut_slice(), flush)?;
-        let new_in = (self.compressor.total_in() - old_in) as usize;
-        let new_out = (self.compressor.total_out() - old_out) as usize;
-        self.buffer.add_size(new_out);
-
-        Ok((new_in, res))
-    }
-}
-
-impl DeflateEncoder<Vec<u8>> {
-    // assume small files
-    pub async fn compress_vec<R>(&mut self, reader: &mut R, size_hint: usize) -> Result<(), Error>
-    where
-        R: AsyncRead + Unpin,
-    {
-        let mut buffer = Vec::with_capacity(size_hint);
-        reader.read_to_end(&mut buffer).await?;
-        self.inner.reserve(size_hint); // should be enough since we want smalller files
-        self.compressor.compress_vec(&buffer[..], &mut self.inner, FlushCompress::Finish)?;
-        Ok(())
-    }
-}
-
-impl<T: AsyncWrite + Unpin> DeflateEncoder<T> {
-    pub async fn compress<R>(&mut self, reader: &mut R) -> Result<(), Error>
-    where
-        R: AsyncRead + Unpin,
-    {
-        let mut buffer = ByteBuffer::with_capacity(BUFFER_SIZE);
-        let mut eof = false;
-        loop {
-            if !eof && !buffer.is_full() {
-                let read = buffer.read_from_async(reader).await?;
-                if read == 0 {
-                    eof = true;
-                }
-            }
-            let (read, _res) = self.encode(&buffer[..], FlushCompress::None)?;
-            buffer.consume(read);
-
-            self.inner.write_all(&self.buffer[..]).await?;
-            self.buffer.clear();
-
-            if buffer.is_empty() && eof {
-                break;
-            }
-        }
-
-        loop {
-            let (_read, res) = self.encode(&[][..], FlushCompress::Finish)?;
-            self.inner.write_all(&self.buffer[..]).await?;
-            self.buffer.clear();
-            if res == flate2::Status::StreamEnd {
-                break;
-            }
-        }
-
-        Ok(())
-    }
-}
-
-impl<T, O> Stream for DeflateEncoder<T>
-where
-    T: Stream<Item = Result<O, io::Error>> + Unpin,
-    O: Into<Bytes>
-{
-    type Item = Result<Bytes, io::Error>;
-
-    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        let this = self.get_mut();
-
-        loop {
-            match this.state {
-                EncoderState::Reading => {
-                    if let Some(res) = ready!(Pin::new(&mut this.inner).poll_next(cx)) {
-                        let buf = res?;
-                        this.input_buffer = buf.into();
-                        this.state = EncoderState::Writing;
-                    } else {
-                        this.state = EncoderState::Flushing;
-                    }
-                }
-                EncoderState::Writing => {
-                    if this.input_buffer.is_empty() {
-                        return Poll::Ready(Some(Err(io_format_err!("empty input during write"))));
-                    }
-                    let mut buf = this.input_buffer.split_off(0);
-                    let (read, res) = this.encode(&buf[..], FlushCompress::None)?;
-                    this.input_buffer = buf.split_off(read);
-                    if this.input_buffer.is_empty() {
-                        this.state = EncoderState::Reading;
-                    }
-                    if this.buffer.is_full() || res == flate2::Status::BufError {
-                        let bytes = this.buffer.remove_data(this.buffer.len()).to_vec();
-                        return Poll::Ready(Some(Ok(bytes.into())));
-                    }
-                }
-                EncoderState::Flushing => {
-                    let (_read, res) = this.encode(&[][..], FlushCompress::Finish)?;
-                    if !this.buffer.is_empty() {
-                        let bytes = this.buffer.remove_data(this.buffer.len()).to_vec();
-                        return Poll::Ready(Some(Ok(bytes.into())));
-                    }
-                    if res == flate2::Status::StreamEnd {
-                        this.state = EncoderState::Finished;
-                    }
-                }
-                EncoderState::Finished => return Poll::Ready(None),
-            }
-        }
-    }
-}

src/tools/mod.rs

@@ -2,8 +2,6 @@
 //!
 //! This is a collection of small and useful tools.
 use std::any::Any;
-use std::collections::HashMap;
-use std::hash::BuildHasher;
 use std::fs::File;
 use std::io::{self, BufRead};
 use std::os::unix::io::RawFd;
@@ -29,7 +27,6 @@ pub use pbs_tools::process_locker::{
     ProcessLocker, ProcessLockExclusiveGuard, ProcessLockSharedGuard
 };
 
-pub mod acl;
 pub mod apt;
 pub mod async_io;
 pub mod compression;
@@ -51,8 +48,6 @@ pub mod statistics;
 pub mod subscription;
 pub mod systemd;
 pub mod ticket;
-pub mod xattr;
-pub mod zip;
 pub mod sgutils2;
 pub mod paperkey;
 
@@ -69,6 +64,7 @@ mod file_logger;
 pub use file_logger::{FileLogger, FileLogOptions};
 
 pub use pbs_tools::broadcast_future::{BroadcastData, BroadcastFuture};
+pub use pbs_tools::ops::ControlFlow;
 
 /// The `BufferedRead` trait provides a single function
 /// `buffered_read`. It returns a reference to an internal buffer. The
@@ -122,65 +118,6 @@ pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<&'a [
     }
 }
 
-pub fn complete_file_name<S>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String>
-where
-    S: BuildHasher,
-{
-    let mut result = vec![];
-
-    use nix::fcntl::AtFlags;
-    use nix::fcntl::OFlag;
-    use nix::sys::stat::Mode;
-
-    let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
-
-    let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
-        Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
-        Err(_) => false,
-    };
-
-    if !is_dir {
-        if let Some(parent) = dirname.parent() {
-            dirname = parent.to_owned();
-        }
-    }
-
-    let mut dir =
-        match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
-            Ok(d) => d,
-            Err(_) => return result,
-        };
-
-    for item in dir.iter() {
-        if let Ok(entry) = item {
-            if let Ok(name) = entry.file_name().to_str() {
-                if name == "." || name == ".." {
-                    continue;
-                }
-                let mut newpath = dirname.clone();
-                newpath.push(name);
-
-                if let Ok(stat) =
-                    nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
-                {
-                    if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
-                        newpath.push("");
-                        if let Some(newpath) = newpath.to_str() {
-                            result.push(newpath.to_owned());
-                        }
-                        continue;
-                    }
-                }
-                if let Some(newpath) = newpath.to_str() {
-                    result.push(newpath.to_owned());
-                }
-            }
-        }
-    }
-
-    result
-}
-
 /// Shortcut for md5 sums.
 pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
     hash(MessageDigest::md5(), data).map_err(Error::from)
@@ -373,17 +310,6 @@ pub fn setup_safe_path_env() {
     }
 }
 
-pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
-    let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
-        Some(n) => &line[n..],
-        None => return &[],
-    };
-    match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
-        Some(n) => &line[..(line.len() - n)],
-        None => &[],
-    }
-}
-
 /// Create the base run-directory.
 ///
 /// This exists to fixate the permissions for the run *base* directory while allowing intermediate
@@ -396,14 +322,3 @@ pub fn create_run_dir() -> Result<(), Error> {
     let _: bool = create_path(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), None, Some(opts))?;
     Ok(())
 }
-
-/// Modeled after the nightly `std::ops::ControlFlow`.
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub enum ControlFlow<B, C = ()> {
-    Continue(C),
-    Break(B),
-}
-
-impl<B> ControlFlow<B> {
-    pub const CONTINUE: ControlFlow<B, ()> = ControlFlow::Continue(());
-}

tests/catar.rs

@@ -1,7 +1,8 @@
+use std::process::Command;
+
 use anyhow::{Error};
 
-use std::process::Command;
+use pbs_client::pxar::*;
-use proxmox_backup::pxar::*;
 
 fn run_test(dir_name: &str) -> Result<(), Error> {