bump version to 0.8.0-1

It is always build for html, but not linked if the devbuild tag isn't
set. This tag is set in the Makefile if the $(BUILD_MODE) variable
isn't "release".

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "proxmox-backup"
-version = "0.5.0"
+version = "0.8.0"
 authors = ["Dietmar Maurer <dietmar@proxmox.com>"]
 edition = "2018"
 license = "AGPL-3"
@@ -38,7 +38,7 @@ pam-sys = "0.5"
 percent-encoding = "2.1"
 pin-utils = "0.1.0"
 pathpatterns = "0.1.1"
-proxmox = { version = "0.1.41", features = [ "sortable-macro", "api-macro" ] }
+proxmox = { version = "0.1.42", features = [ "sortable-macro", "api-macro" ] }
 #proxmox = { git = "ssh://gitolite3@proxdev.maurer-it.com/rust/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
 #proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro" ] }
 proxmox-fuse = "0.1.0"

Makefile

@@ -37,6 +37,8 @@ CARGO ?= cargo
 COMPILED_BINS := \
 	$(addprefix $(COMPILEDIR)/,$(USR_BIN) $(USR_SBIN) $(SERVICE_BIN))
 
+export DEB_VERSION DEB_VERSION_UPSTREAM
+
 SERVER_DEB=${PACKAGE}-server_${DEB_VERSION}_${ARCH}.deb
 CLIENT_DEB=${PACKAGE}-client_${DEB_VERSION}_${ARCH}.deb
 DOC_DEB=${PACKAGE}-docs_${DEB_VERSION}_all.deb

debian/changelog

@@ -1,3 +1,45 @@
+rust-proxmox-backup (0.8.0-1) unstable; urgency=medium
+
+  * implement  get_runtime_with_builder
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 07 Jul 2020 10:15:26 +0200
+
+rust-proxmox-backup (0.7.0-1) unstable; urgency=medium
+
+  *  implement clone for RemoteChunkReader
+
+  * improve docs
+
+  * client: add --encryption boolen parameter
+
+  * client: use default encryption key if it is available
+
+  * d/rules: do not compress .pdf files
+
+  * ui: various fixes
+
+  * add beta text with link to bugtracker
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 07 Jul 2020 07:40:05 +0200
+
+rust-proxmox-backup (0.6.0-1) unstable; urgency=medium
+
+  * make ReadChunk not require mutable self.
+
+  * ui: increase timeout for snapshot listing
+
+  * ui: consistently spell Datastore without space between words
+
+  * ui: disk create: sync and improve 'add-datastore' checkbox label
+
+  * proxmox-backup-client: add benchmark command
+
+  * pxar: fixup 'vanished-file' logic a bit
+
+  * ui: add verify button
+
+ -- Proxmox Support Team <support@proxmox.com>  Fri, 03 Jul 2020 09:45:52 +0200
+
 rust-proxmox-backup (0.5.0-1) unstable; urgency=medium
 
   * partially revert commit 1f82f9b7b5d231da22a541432d5617cb303c0000

debian/control.in

@@ -3,11 +3,14 @@ Architecture: any
 Depends: fonts-font-awesome,
          libjs-extjs (>= 6.0.1),
          libzstd1 (>= 1.3.8),
+         lvm2,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
          proxmox-widget-toolkit (>= 2.2-4),
+         smartmontools,
          ${misc:Depends},
          ${shlibs:Depends},
+Recommends: zfsutils-linux,
 Description: Proxmox Backup Server daemon with tools and GUI
  This package contains the Proxmox Backup Server daemons and related
  tools. This includes a web-based graphical user interface.

debian/rules

@@ -45,3 +45,6 @@ override_dh_installsystemd:
 # TODO: remove once available (Debian 11 ?)
 override_dh_dwz:
 	dh_dwz --no-dwz-multifile
+
+override_dh_compress:
+	dh_compress -X.pdf

docs/Makefile

@@ -1,11 +1,5 @@
 include ../defines.mk
 
-ifeq ($(BUILD_MODE), release)
-COMPILEDIR := ../target/release
-else
-COMPILEDIR := ../target/debug
-endif
-
 GENERATED_SYNOPSIS := 						\
 	proxmox-backup-client/synopsis.rst			\
 	proxmox-backup-client/catalog-shell-synopsis.rst 	\
@@ -26,6 +20,15 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 BUILDDIR      = output
 
+ifeq ($(BUILD_MODE), release)
+COMPILEDIR := ../target/release
+SPHINXOPTS    += -t release
+else
+COMPILEDIR := ../target/debug
+SPHINXOPTS    += -t devbuild
+endif
+
+
 # Sphinx internal variables.
 ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) .
 

docs/conf.py

@@ -17,7 +17,7 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
+import os
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
@@ -45,8 +45,11 @@ PygmentsBridge.latex_formatter = CustomLatexFormatter
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
+
 extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo"]
 
+todo_link_only = True
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -76,9 +79,11 @@ author = 'Proxmox Support Team'
 # built documents.
 #
 # The short X.Y version.
-version = '0.2'
+vstr = lambda s: '<devbuild>' if s is None else str(s)
+
+version = vstr(os.getenv('DEB_VERSION_UPSTREAM'))
 # The full version, including alpha/beta/rc tags.
-release = '0.2-1'
+release = vstr(os.getenv('DEB_VERSION'))
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/index.rst

@@ -1,18 +1,15 @@
 .. Proxmox Backup documentation master file
 
-Welcome to Proxmox Backup's documentation!
+Welcome to the Proxmox Backup documentation!
-==========================================
+============================================
 
-Copyright (C) 2019 Proxmox Server Solutions GmbH
+Copyright (C) 2019-2020 Proxmox Server Solutions GmbH
 
-Permission is granted to copy, distribute and/or modify this document
+Permission is granted to copy, distribute and/or modify this document under the
-under the terms of the GNU Free Documentation License, Version 1.3 or
+terms of the GNU Free Documentation License, Version 1.3 or any later version
-any later version published by the Free Software Foundation; with no
+published by the Free Software Foundation; with no Invariant Sections, no
-Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A
+Front-Cover Texts, and no Back-Cover Texts. A copy of the license is included
-copy of the license is included in the section entitled "GNU Free
+in the section entitled "GNU Free Documentation License".
-Documentation License".
-
-.. todolist::
 
 
 .. toctree::
@@ -37,5 +34,14 @@ Documentation License".
    glossary.rst
    GFDL.rst
 
+.. only:: html and devbuild
+
+    .. toctree::
+       :maxdepth: 2
+       :caption: Developer Appendix
+
+       todos.rst
+
+
 * :ref:`genindex`
 

docs/introduction.rst

@@ -1,57 +1,61 @@
 Introduction
 ============
 
-This documentation is written in :term:`reStructuredText` and formatted with :term:`Sphinx`.
+This documentation is written in :term:`reStructuredText` and formatted with
+:term:`Sphinx`.
 
 
-What is Proxmox Backup
+What is Proxmox Backup Server
-----------------------
+-----------------------------
 
-Proxmox Backup is an enterprise class client-server backup software,
+Proxmox Backup Server is an enterprise-class client-server backup software that
-specially optimized for the `Proxmox Virtual Environment`_ to backup
+backups :term:`virtual machine`\ s, :term:`container`\ s, and physical hosts.
-:term:`virtual machine`\ s and :term:`container`\ s. It is also
+It is specially optimized for the `Proxmox Virtual Environment`_ platform and
-possible to backup physical hosts.
+allows you to backup your data securely, even between remote sites, providing
+easy management with a web-based user interface.
 
-It supports deduplication, compression and authenticated encryption
+Proxmox Backup Server supports deduplication, compression, and authenticated
-(AE_). Using :term:`Rust` as implementation language guarantees high
+encryption (AE_). Using :term:`Rust` as implementation language guarantees high
 performance, low resource usage, and a safe, high quality code base.
 
-Encryption is done at the client side. This makes backups to not fully
+It features strong encryption done on the client side. Thus, it's possible to
-trusted targets possible.
+backup data to not fully trusted targets.
 
 
 Architecture
 ------------
 
-Proxmox Backup uses a `Client-server model`_. The server is
+Proxmox Backup Server uses a `client-server model`_. The server stores the
-responsible to store the backup data and provides an API to create
+backup data and provides an API to create backups and restore data. With the
-backups and restore data. It is possible to manage disks and
+API it's also possible to manage disks and other server side resources.
-other server side resources using this API.
 
-A backup client uses this API to access the backed up data,
+The backup client uses this API to access the backed up data. With the command
-i.e. ``proxmox-backup-client`` is a command line tool to create
+line tool ``proxmox-backup-client`` you can create backups and restore data.
-backups and restore data. We deliver an integrated client for
+For QEMU_ with `Proxmox Virtual Environment`_ we deliver an integrated client.
-QEMU_ with `Proxmox Virtual Environment`_.
 
-A single backup is allowed to contain several archives. For example,
+A single backup is allowed to contain several archives. For example, when you
-when you backup a :term:`virtual machine`, each disk is stored as a
+backup a :term:`virtual machine`, each disk is stored as a separate archive
-separate archive inside that backup. The VM configuration also gets an
+inside that backup. The VM configuration itself is stored as an extra file.
-extra file. This way, it is easy to access and restore important parts
+This way, it is easy to access and restore only important parts of the backup
-of the backup without having to scan the whole backup.
+without the need to scan the whole backup.
 
 
 Main Features
 -------------
 
-:Proxmox VE: The `Proxmox Virtual Environment`_ is fully
+:Support for Proxmox VE: The `Proxmox Virtual Environment`_ is fully
-   supported. You can backup :term:`virtual machine`\ s and
+   supported and you can easily backup :term:`virtual machine`\ s and
    :term:`container`\ s.
 
-:GUI: We provide a graphical, web based user interface.
+:Performance: The whole software stack is written in :term:`Rust`,
+   to provide high speed and memory efficiency.
 
-:Deduplication: Incremental backups produce large amounts of duplicate
+:Deduplication: Periodic backups produce large amounts of duplicate
-   data. The deduplication layer removes that redundancy and makes
+   data. The deduplication layer avoids redundancy and minimizes the used
-   incremental backups small and space efficient.
+   storage space.
+
+:Incremental backups: Changes between backups are typically low. Reading and
+   sending only the delta reduces storage and network impact of backups.
 
 :Data Integrity: The built in `SHA-256`_ checksum algorithm assures the
    accuracy and consistency of your backups.
@@ -59,43 +63,43 @@ Main Features
 :Remote Sync: It is possible to efficiently synchronize data to remote
    sites. Only deltas containing new data are transferred.
 
-:Performance: The whole software stack is written in :term:`Rust`,
+:Compression: The ultra fast Zstandard_ compression is able to compress
-   to provide high speed and memory efficiency.
-
-:Compression: Ultra fast Zstandard_ compression is able to compress
    several gigabytes of data per second.
 
-:Encryption: Backups can be encrypted client-side using AES-256 in
+:Encryption: Backups can be encrypted on the client-side using AES-256 in
    GCM_ mode. This authenticated encryption mode (AE_) provides very
    high performance on modern hardware.
 
-:Open Source: No secrets. You have access to all the source code.
+:Web interface: Manage Proxmox backups with the integrated web-based user
+   interface.
 
-:Support: Commercial support options are available from `Proxmox`_.
+:Open Source: No secrets. Proxmox Backup Server is free and open-source
+ software. The source code is licensed under AGPL, v3.
+
+:Support: Enterprise support is available from `Proxmox`_.
 
 
-Why Backup?
+Reasons for Data Backup?
------------
+------------------------
 
-The primary purpose of a backup is to protect against data loss. Data
+The main purpose of a backup is to protect against data loss. Data loss can be
-loss can be caused by faulty hardware, but also by human error.
+caused by faulty hardware but also by human error.
 
-A common mistake is to delete a file or folder which is still
+A common mistake is to accidentally delete a file or folder which is still
-required. Virtualization can amplify this problem. It is now
+required. Virtualization can even amplify this problem; it easily happens that
-easy to delete a whole virtual machine by pressing a single button.
+a whole virtual machine is deleted by just pressing a single button.
 
-Backups can serve as a toolkit for administrators to temporarily
+For administrators, backups can serve as a useful toolkit for temporarily
-store data. For example, it is common practice to perform full backups
+storing data. For example, it is common practice to perform full backups before
-before installing major software updates. If something goes wrong, you
+installing major software updates. If something goes wrong, you can easily
-can restore the previous state.
+restore the previous state.
 
-Another reason for backups are legal requirements. Some data must be
+Another reason for backups are legal requirements. Some data, especially
-kept in a safe place for several years by law, so that it can be accessed if
+business records, must be kept in a safe place for several years by law, so
-required.
+that they can be accessed if required.
 
-Data loss can be very costly as it can severely restrict your
+In general, data loss is very costly as it can severely damage your business.
-business. Therefore, make sure that you perform a backup regularly
+Therefore, ensure that you perform regular backups and run restore tests.
-and run restore tests.
 
 
 Software Stack
@@ -107,14 +111,14 @@ Software Stack
 License
 -------
 
-Copyright (C) 2019 Proxmox Server Solutions GmbH
+Copyright (C) 2019-2020 Proxmox Server Solutions GmbH
 
 This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
 
-Proxmox Backup is free software: you can redistribute it and/or modify
+Proxmox Backup Server is free and open source software: you can use it,
-it under the terms of the GNU Affero General Public License as
+redistribute it, and/or modify it under the terms of the GNU Affero General
-published by the Free Software Foundation, either version 3 of the
+Public License as published by the Free Software Foundation, either version 3
-License, or (at your option) any later version.
+of the License, or (at your option) any later version.
 
 This program is distributed in the hope that it will be useful, but
 ``WITHOUT ANY WARRANTY``; without even the implied warranty of

src/api2/node/disks.rs

@@ -26,10 +26,10 @@ pub mod zfs;
                 schema: NODE_SCHEMA,
             },
             skipsmart: {
-		description: "Skip smart checks.",
+                description: "Skip smart checks.",
-		type: bool,
+                type: bool,
-		optional: true,
+                optional: true,
-		default: false,
+                default: false,
             },
             "usage-type": {
                 type: DiskUsageType,

src/backup/async_index_reader.rs

@@ -36,7 +36,7 @@ impl<S: AsyncReadChunk, I: IndexFile> AsyncIndexReader<S, I> {
         Self {
             store: Some(store),
             index,
-            read_buffer: Vec::with_capacity(1024*1024),
+            read_buffer: Vec::with_capacity(1024 * 1024),
             current_chunk_idx: 0,
             current_chunk_digest: [0u8; 32],
             state: AsyncIndexReaderState::NoData,
@@ -44,9 +44,10 @@ impl<S: AsyncReadChunk, I: IndexFile> AsyncIndexReader<S, I> {
     }
 }
 
-impl<S, I> AsyncRead for AsyncIndexReader<S, I> where
+impl<S, I> AsyncRead for AsyncIndexReader<S, I>
-S: AsyncReadChunk + Unpin + Sync + 'static,
+where
-I: IndexFile + Unpin
+    S: AsyncReadChunk + Unpin + Sync + 'static,
+    I: IndexFile + Unpin,
 {
     fn poll_read(
         self: Pin<&mut Self>,
@@ -57,7 +58,7 @@ I: IndexFile + Unpin
         loop {
             match &mut this.state {
                 AsyncIndexReaderState::NoData => {
-                    if this.current_chunk_idx >= this.index.index_count()  {
+                    if this.current_chunk_idx >= this.index.index_count() {
                         return Poll::Ready(Ok(0));
                     }
 
@@ -67,7 +68,7 @@ I: IndexFile + Unpin
                         .ok_or(io_format_err!("could not get digest"))?
                         .clone();
 
-                    if digest  == this.current_chunk_digest {
+                    if digest == this.current_chunk_digest {
                         this.state = AsyncIndexReaderState::HaveData(0);
                         continue;
                     }
@@ -78,7 +79,7 @@ I: IndexFile + Unpin
                         Some(store) => store,
                         None => {
                             return Poll::Ready(Err(io_format_err!("could not find store")));
-                        },
+                        }
                     };
 
                     let future = async move {
@@ -88,7 +89,7 @@ I: IndexFile + Unpin
                     };
 
                     this.state = AsyncIndexReaderState::WaitForData(future.boxed());
-                },
+                }
                 AsyncIndexReaderState::WaitForData(ref mut future) => {
                     match ready!(future.as_mut().poll(cx)) {
                         Ok((store, mut chunk_data)) => {
@@ -96,12 +97,12 @@ I: IndexFile + Unpin
                             this.read_buffer.append(&mut chunk_data);
                             this.state = AsyncIndexReaderState::HaveData(0);
                             this.store = Some(store);
-                        },
+                        }
                         Err(err) => {
                             return Poll::Ready(Err(io_err_other(err)));
-                        },
+                        }
                     };
-                },
+                }
                 AsyncIndexReaderState::HaveData(offset) => {
                     let offset = *offset;
                     let len = this.read_buffer.len();
@@ -111,7 +112,7 @@ I: IndexFile + Unpin
                         buf.len()
                     };
 
-                    buf[0..n].copy_from_slice(&this.read_buffer[offset..offset+n]);
+                    buf[0..n].copy_from_slice(&this.read_buffer[offset..(offset + n)]);
                     if offset + n == len {
                         this.state = AsyncIndexReaderState::NoData;
                         this.current_chunk_idx += 1;
@@ -120,7 +121,7 @@ I: IndexFile + Unpin
                     }
 
                     return Poll::Ready(Ok(n));
-                },
+                }
             }
         }
     }

src/bin/proxmox-backup-client.rs

@@ -1,6 +1,5 @@
 use std::collections::{HashSet, HashMap};
 use std::io::{self, Write, Seek, SeekFrom};
-use std::os::unix::fs::OpenOptionsExt;
 use std::path::{Path, PathBuf};
 use std::pin::Pin;
 use std::sync::{Arc, Mutex};
@@ -15,9 +14,7 @@ use tokio::sync::mpsc;
 use xdg::BaseDirectories;
 
 use pathpatterns::{MatchEntry, MatchType, PatternFlag};
-use proxmox::{sortable, identity};
 use proxmox::tools::fs::{file_get_contents, file_get_json, replace_file, CreateOptions, image_size};
-use proxmox::sys::linux::tty;
 use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment};
 use proxmox::api::schema::*;
 use proxmox::api::cli::*;
@@ -30,9 +27,7 @@ use proxmox_backup::client::*;
 use proxmox_backup::pxar::catalog::*;
 use proxmox_backup::backup::{
     archive_type,
-    encrypt_key_with_passphrase,
     load_and_decrypt_key,
-    store_key_config,
     verify_chunk_size,
     ArchiveType,
     AsyncReadChunk,
@@ -50,7 +45,6 @@ use proxmox_backup::backup::{
     FixedChunkStream,
     FixedIndexReader,
     IndexFile,
-    KeyConfig,
     MANIFEST_BLOB_NAME,
     Shell,
 };
@@ -71,6 +65,11 @@ pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
     "Path to encryption key. All data will be encrypted using this key.")
     .schema();
 
+pub const ENCRYPTION_SCHEMA: Schema = BooleanSchema::new(
+    "Explicitly enable or disable encryption. \
+     (Allows disabling encryption when a default key file is present.)")
+    .schema();
+
 const CHUNK_SIZE_SCHEMA: Schema = IntegerSchema::new(
     "Chunk size in KB. Must be a power of 2.")
     .minimum(64)
@@ -546,79 +545,6 @@ fn api_logout(param: Value) -> Result<Value, Error> {
     Ok(Value::Null)
 }
 
-#[api(
-   input: {
-        properties: {
-            repository: {
-                schema: REPO_URL_SCHEMA,
-                optional: true,
-            },
-            snapshot: {
-                type: String,
-                description: "Snapshot path.",
-             },
-        }
-   }
-)]
-/// Dump catalog.
-async fn dump_catalog(param: Value) -> Result<Value, Error> {
-
-    let repo = extract_repository_from_value(&param)?;
-
-    let path = tools::required_string_param(&param, "snapshot")?;
-    let snapshot: BackupDir = path.parse()?;
-
-    let keyfile = param["keyfile"].as_str().map(PathBuf::from);
-
-    let crypt_config = match keyfile {
-        None => None,
-        Some(path) => {
-            let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
-            Some(Arc::new(CryptConfig::new(key)?))
-        }
-    };
-
-    let client = connect(repo.host(), repo.user())?;
-
-    let client = BackupReader::start(
-        client,
-        crypt_config.clone(),
-        repo.store(),
-        &snapshot.group().backup_type(),
-        &snapshot.group().backup_id(),
-        snapshot.backup_time(),
-        true,
-    ).await?;
-
-    let manifest = client.download_manifest().await?;
-
-    let index = client.download_dynamic_index(&manifest, CATALOG_NAME).await?;
-
-    let most_used = index.find_most_used_chunks(8);
-
-    let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
-
-    let mut reader = BufferedDynamicReader::new(index, chunk_reader);
-
-    let mut catalogfile = std::fs::OpenOptions::new()
-        .write(true)
-        .read(true)
-        .custom_flags(libc::O_TMPFILE)
-        .open("/tmp")?;
-
-    std::io::copy(&mut reader, &mut catalogfile)
-        .map_err(|err| format_err!("unable to download catalog - {}", err))?;
-
-    catalogfile.seek(SeekFrom::Start(0))?;
-
-    let mut catalog_reader = CatalogReader::new(catalogfile);
-
-    catalog_reader.dump()?;
-
-    record_repository(&repo);
-
-    Ok(Value::Null)
-}
 
 #[api(
    input: {
@@ -738,6 +664,37 @@ fn spawn_catalog_upload(
     Ok((catalog, catalog_result_rx))
 }
 
+fn keyfile_parameters(param: &Value) -> Result<Option<PathBuf>, Error> {
+    Ok(match (param.get("keyfile"), param.get("encryption")) {
+        // no parameters:
+        (None, None) => key::optional_default_key_path()?,
+
+        // just --encryption=false
+        (None, Some(Value::Bool(false))) => None,
+
+        // just --encryption=true
+        (None, Some(Value::Bool(true))) => match key::optional_default_key_path()? {
+            None => bail!("--encryption=false without --keyfile and no default key file available"),
+            Some(path) => Some(path),
+        }
+
+        // just --keyfile
+        (Some(Value::String(keyfile)), None) => Some(PathBuf::from(keyfile)),
+
+        // --keyfile and --encryption=false
+        (Some(Value::String(_)), Some(Value::Bool(false))) => {
+            bail!("--keyfile and --encryption=false are mutually exclusive");
+        }
+
+        // --keyfile and --encryption=true
+        (Some(Value::String(keyfile)), Some(Value::Bool(true))) => Some(PathBuf::from(keyfile)),
+
+        // wrong value types:
+        (Some(_), _) => bail!("bad --keyfile parameter"),
+        (_, Some(_)) => bail!("bad --encryption parameter"),
+    })
+}
+
 #[api(
    input: {
        properties: {
@@ -764,6 +721,10 @@ fn spawn_catalog_upload(
                schema: KEYFILE_SCHEMA,
                optional: true,
            },
+           encryption: {
+               schema: ENCRYPTION_SCHEMA,
+               optional: true,
+           },
            "skip-lost-and-found": {
                type: Boolean,
                description: "Skip lost+found directory.",
@@ -833,7 +794,7 @@ async fn create_backup(
         verify_chunk_size(size)?;
     }
 
-    let keyfile = param["keyfile"].as_str().map(PathBuf::from);
+    let keyfile = keyfile_parameters(&param)?;
 
     let backup_id = param["backup-id"].as_str().unwrap_or(&proxmox::tools::nodename());
 
@@ -935,7 +896,7 @@ async fn create_backup(
     let (crypt_config, rsa_encrypted_key) = match keyfile {
         None => (None, None),
         Some(path) => {
-            let (key, created) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
+            let (key, created) = load_and_decrypt_key(&path, &key::get_encryption_key_password)?;
 
             let crypt_config = CryptConfig::new(key)?;
 
@@ -1198,6 +1159,10 @@ We do not extraxt '.pxar' archives when writing to standard output.
                schema: KEYFILE_SCHEMA,
                optional: true,
            },
+           encryption: {
+               schema: ENCRYPTION_SCHEMA,
+               optional: true,
+           },
        }
    }
 )]
@@ -1228,12 +1193,12 @@ async fn restore(param: Value) -> Result<Value, Error> {
     let target = tools::required_string_param(&param, "target")?;
     let target = if target == "-" { None } else { Some(target) };
 
-    let keyfile = param["keyfile"].as_str().map(PathBuf::from);
+    let keyfile = keyfile_parameters(&param)?;
 
     let crypt_config = match keyfile {
         None => None,
         Some(path) => {
-            let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
+            let (key, _) = load_and_decrypt_key(&path, &key::get_encryption_key_password)?;
             Some(Arc::new(CryptConfig::new(key)?))
         }
     };
@@ -1358,6 +1323,10 @@ async fn restore(param: Value) -> Result<Value, Error> {
                schema: KEYFILE_SCHEMA,
                optional: true,
            },
+           encryption: {
+               schema: ENCRYPTION_SCHEMA,
+               optional: true,
+           },
        }
    }
 )]
@@ -1372,12 +1341,12 @@ async fn upload_log(param: Value) -> Result<Value, Error> {
 
     let mut client = connect(repo.host(), repo.user())?;
 
-    let keyfile = param["keyfile"].as_str().map(PathBuf::from);
+    let keyfile = keyfile_parameters(&param)?;
 
     let crypt_config = match keyfile {
         None => None,
         Some(path) => {
-            let (key, _created) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
+            let (key, _created) = load_and_decrypt_key(&path, &key::get_encryption_key_password)?;
             let crypt_config = CryptConfig::new(key)?;
             Some(Arc::new(crypt_config))
         }
@@ -1742,69 +1711,6 @@ fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<Stri
     result
 }
 
-fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
-
-    // fixme: implement other input methods
-
-    use std::env::VarError::*;
-    match std::env::var("PBS_ENCRYPTION_PASSWORD") {
-        Ok(p) => return Ok(p.as_bytes().to_vec()),
-        Err(NotUnicode(_)) => bail!("PBS_ENCRYPTION_PASSWORD contains bad characters"),
-        Err(NotPresent) => {
-            // Try another method
-        }
-    }
-
-    // If we're on a TTY, query the user for a password
-    if tty::stdin_isatty() {
-        return Ok(tty::read_password("Encryption Key Password: ")?);
-    }
-
-    bail!("no password input mechanism available");
-}
-
-fn key_create(
-    param: Value,
-    _info: &ApiMethod,
-    _rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
-
-    let path = tools::required_string_param(&param, "path")?;
-    let path = PathBuf::from(path);
-
-    let kdf = param["kdf"].as_str().unwrap_or("scrypt");
-
-    let key = proxmox::sys::linux::random_data(32)?;
-
-    if kdf == "scrypt" {
-        // always read passphrase from tty
-        if !tty::stdin_isatty() {
-            bail!("unable to read passphrase - no tty");
-        }
-
-        let password = tty::read_and_verify_password("Encryption Key Password: ")?;
-
-        let key_config = encrypt_key_with_passphrase(&key, &password)?;
-
-        store_key_config(&path, false, key_config)?;
-
-        Ok(Value::Null)
-    } else if kdf == "none" {
-        let created =  Local.timestamp(Local::now().timestamp(), 0);
-
-        store_key_config(&path, false, KeyConfig {
-            kdf: None,
-            created,
-            modified: created,
-            data: key,
-        })?;
-
-        Ok(Value::Null)
-    } else {
-        unreachable!();
-    }
-}
-
 fn master_pubkey_path() -> Result<PathBuf, Error> {
     let base = BaseDirectories::with_prefix("proxmox-backup")?;
 
@@ -1814,176 +1720,6 @@ fn master_pubkey_path() -> Result<PathBuf, Error> {
     Ok(path)
 }
 
-fn key_import_master_pubkey(
-    param: Value,
-    _info: &ApiMethod,
-    _rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
-
-    let path = tools::required_string_param(&param, "path")?;
-    let path = PathBuf::from(path);
-
-    let pem_data = file_get_contents(&path)?;
-
-    if let Err(err) = openssl::pkey::PKey::public_key_from_pem(&pem_data) {
-        bail!("Unable to decode PEM data - {}", err);
-    }
-
-    let target_path = master_pubkey_path()?;
-
-    replace_file(&target_path, &pem_data, CreateOptions::new())?;
-
-    println!("Imported public master key to {:?}", target_path);
-
-    Ok(Value::Null)
-}
-
-fn key_create_master_key(
-    _param: Value,
-    _info: &ApiMethod,
-    _rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
-
-    // we need a TTY to query the new password
-    if !tty::stdin_isatty() {
-        bail!("unable to create master key - no tty");
-    }
-
-    let rsa = openssl::rsa::Rsa::generate(4096)?;
-    let pkey = openssl::pkey::PKey::from_rsa(rsa)?;
-
-
-    let password = String::from_utf8(tty::read_and_verify_password("Master Key Password: ")?)?;
-
-    let pub_key: Vec<u8> = pkey.public_key_to_pem()?;
-    let filename_pub = "master-public.pem";
-    println!("Writing public master key to {}", filename_pub);
-    replace_file(filename_pub, pub_key.as_slice(), CreateOptions::new())?;
-
-    let cipher = openssl::symm::Cipher::aes_256_cbc();
-    let priv_key: Vec<u8> = pkey.private_key_to_pem_pkcs8_passphrase(cipher, password.as_bytes())?;
-
-    let filename_priv = "master-private.pem";
-    println!("Writing private master key to {}", filename_priv);
-    replace_file(filename_priv, priv_key.as_slice(), CreateOptions::new())?;
-
-    Ok(Value::Null)
-}
-
-fn key_change_passphrase(
-    param: Value,
-    _info: &ApiMethod,
-    _rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
-
-    let path = tools::required_string_param(&param, "path")?;
-    let path = PathBuf::from(path);
-
-    let kdf = param["kdf"].as_str().unwrap_or("scrypt");
-
-    // we need a TTY to query the new password
-    if !tty::stdin_isatty() {
-        bail!("unable to change passphrase - no tty");
-    }
-
-    let (key, created) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
-
-    if kdf == "scrypt" {
-
-        let password = tty::read_and_verify_password("New Password: ")?;
-
-        let mut new_key_config = encrypt_key_with_passphrase(&key, &password)?;
-        new_key_config.created = created; // keep original value
-
-        store_key_config(&path, true, new_key_config)?;
-
-        Ok(Value::Null)
-    } else if kdf == "none" {
-        let modified =  Local.timestamp(Local::now().timestamp(), 0);
-
-        store_key_config(&path, true, KeyConfig {
-            kdf: None,
-            created, // keep original value
-            modified,
-            data: key.to_vec(),
-        })?;
-
-        Ok(Value::Null)
-    } else {
-        unreachable!();
-    }
-}
-
-fn key_mgmt_cli() -> CliCommandMap {
-
-    const KDF_SCHEMA: Schema =
-        StringSchema::new("Key derivation function. Choose 'none' to store the key unecrypted.")
-        .format(&ApiStringFormat::Enum(&[
-            EnumEntry::new("scrypt", "SCrypt"),
-            EnumEntry::new("none", "Do not encrypt the key")]))
-        .default("scrypt")
-        .schema();
-
-    #[sortable]
-    const API_METHOD_KEY_CREATE: ApiMethod = ApiMethod::new(
-        &ApiHandler::Sync(&key_create),
-        &ObjectSchema::new(
-            "Create a new encryption key.",
-            &sorted!([
-                ("path", false, &StringSchema::new("File system path.").schema()),
-                ("kdf", true, &KDF_SCHEMA),
-            ]),
-        )
-    );
-
-    let key_create_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE)
-        .arg_param(&["path"])
-        .completion_cb("path", tools::complete_file_name);
-
-    #[sortable]
-    const API_METHOD_KEY_CHANGE_PASSPHRASE: ApiMethod = ApiMethod::new(
-        &ApiHandler::Sync(&key_change_passphrase),
-        &ObjectSchema::new(
-            "Change the passphrase required to decrypt the key.",
-            &sorted!([
-                ("path", false, &StringSchema::new("File system path.").schema()),
-                ("kdf", true, &KDF_SCHEMA),
-            ]),
-        )
-    );
-
-    let key_change_passphrase_cmd_def = CliCommand::new(&API_METHOD_KEY_CHANGE_PASSPHRASE)
-        .arg_param(&["path"])
-        .completion_cb("path", tools::complete_file_name);
-
-    const API_METHOD_KEY_CREATE_MASTER_KEY: ApiMethod = ApiMethod::new(
-        &ApiHandler::Sync(&key_create_master_key),
-        &ObjectSchema::new("Create a new 4096 bit RSA master pub/priv key pair.", &[])
-    );
-
-    let key_create_master_key_cmd_def = CliCommand::new(&API_METHOD_KEY_CREATE_MASTER_KEY);
-
-    #[sortable]
-    const API_METHOD_KEY_IMPORT_MASTER_PUBKEY: ApiMethod = ApiMethod::new(
-        &ApiHandler::Sync(&key_import_master_pubkey),
-        &ObjectSchema::new(
-            "Import a new RSA public key and use it as master key. The key is expected to be in '.pem' format.",
-            &sorted!([ ("path", false, &StringSchema::new("File system path.").schema()) ]),
-        )
-    );
-
-    let key_import_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_KEY_IMPORT_MASTER_PUBKEY)
-        .arg_param(&["path"])
-        .completion_cb("path", tools::complete_file_name);
-
-    CliCommandMap::new()
-        .insert("create", key_create_cmd_def)
-        .insert("create-master-key", key_create_master_key_cmd_def)
-        .insert("import-master-pubkey", key_import_master_pubkey_cmd_def)
-        .insert("change-passphrase", key_change_passphrase_cmd_def)
-}
-
-
 use proxmox_backup::client::RemoteChunkReader;
 /// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
 /// async use!
@@ -2026,140 +1762,6 @@ impl ReadAt for BufferedDynamicReadAt {
     }
 }
 
-
-#[api(
-    input: {
-        properties: {
-            "snapshot": {
-                type: String,
-                description: "Group/Snapshot path.",
-            },
-            "archive-name": {
-                type: String,
-                description: "Backup archive name.",
-            },
-            "repository": {
-                optional: true,
-                schema: REPO_URL_SCHEMA,
-            },
-            "keyfile": {
-                optional: true,
-                type: String,
-                description: "Path to encryption key.",
-            },
-        },
-    },
-)]
-/// Shell to interactively inspect and restore snapshots.
-async fn catalog_shell(param: Value) -> Result<(), Error> {
-    let repo = extract_repository_from_value(&param)?;
-    let client = connect(repo.host(), repo.user())?;
-    let path = tools::required_string_param(&param, "snapshot")?;
-    let archive_name = tools::required_string_param(&param, "archive-name")?;
-
-    let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
-        let group: BackupGroup = path.parse()?;
-        api_datastore_latest_snapshot(&client, repo.store(), group).await?
-    } else {
-        let snapshot: BackupDir = path.parse()?;
-        (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
-    };
-
-    let keyfile = param["keyfile"].as_str().map(|p| PathBuf::from(p));
-    let crypt_config = match keyfile {
-        None => None,
-        Some(path) => {
-            let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
-            Some(Arc::new(CryptConfig::new(key)?))
-        }
-    };
-
-    let server_archive_name = if archive_name.ends_with(".pxar") {
-        format!("{}.didx", archive_name)
-    } else {
-        bail!("Can only mount pxar archives.");
-    };
-
-    let client = BackupReader::start(
-        client,
-        crypt_config.clone(),
-        repo.store(),
-        &backup_type,
-        &backup_id,
-        backup_time,
-        true,
-    ).await?;
-
-    let mut tmpfile = std::fs::OpenOptions::new()
-        .write(true)
-        .read(true)
-        .custom_flags(libc::O_TMPFILE)
-        .open("/tmp")?;
-
-    let manifest = client.download_manifest().await?;
-
-    let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
-    let most_used = index.find_most_used_chunks(8);
-    let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), most_used);
-    let reader = BufferedDynamicReader::new(index, chunk_reader);
-    let archive_size = reader.archive_size();
-    let reader: proxmox_backup::pxar::fuse::Reader =
-        Arc::new(BufferedDynamicReadAt::new(reader));
-    let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
-
-    client.download(CATALOG_NAME, &mut tmpfile).await?;
-    let index = DynamicIndexReader::new(tmpfile)
-        .map_err(|err| format_err!("unable to read catalog index - {}", err))?;
-
-    // Note: do not use values stored in index (not trusted) - instead, computed them again
-    let (csum, size) = index.compute_csum();
-    manifest.verify_file(CATALOG_NAME, &csum, size)?;
-
-    let most_used = index.find_most_used_chunks(8);
-    let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
-    let mut reader = BufferedDynamicReader::new(index, chunk_reader);
-    let mut catalogfile = std::fs::OpenOptions::new()
-        .write(true)
-        .read(true)
-        .custom_flags(libc::O_TMPFILE)
-        .open("/tmp")?;
-
-    std::io::copy(&mut reader, &mut catalogfile)
-        .map_err(|err| format_err!("unable to download catalog - {}", err))?;
-
-    catalogfile.seek(SeekFrom::Start(0))?;
-    let catalog_reader = CatalogReader::new(catalogfile);
-    let state = Shell::new(
-        catalog_reader,
-        &server_archive_name,
-        decoder,
-    ).await?;
-
-    println!("Starting interactive shell");
-    state.shell().await?;
-
-    record_repository(&repo);
-
-    Ok(())
-}
-
-fn catalog_mgmt_cli() -> CliCommandMap {
-    let catalog_shell_cmd_def = CliCommand::new(&API_METHOD_CATALOG_SHELL)
-        .arg_param(&["snapshot", "archive-name"])
-        .completion_cb("repository", complete_repository)
-        .completion_cb("archive-name", complete_pxar_archive_name)
-        .completion_cb("snapshot", complete_group_or_snapshot);
-
-    let catalog_dump_cmd_def = CliCommand::new(&API_METHOD_DUMP_CATALOG)
-        .arg_param(&["snapshot"])
-        .completion_cb("repository", complete_repository)
-        .completion_cb("snapshot", complete_backup_snapshot);
-
-    CliCommandMap::new()
-        .insert("dump", catalog_dump_cmd_def)
-        .insert("shell", catalog_shell_cmd_def)
-}
-
 fn main() {
 
     let backup_cmd_def = CliCommand::new(&API_METHOD_CREATE_BACKUP)
@@ -2235,7 +1837,7 @@ fn main() {
         .insert("snapshots", snapshots_cmd_def)
         .insert("files", files_cmd_def)
         .insert("status", status_cmd_def)
-        .insert("key", key_mgmt_cli())
+        .insert("key", key::cli())
         .insert("mount", mount_cmd_def())
         .insert("catalog", catalog_mgmt_cli())
         .insert("task", task_mgmt_cli())

src/bin/proxmox_backup_client/benchmark.rs

@@ -19,7 +19,6 @@ use proxmox_backup::client::*;
 use crate::{
     KEYFILE_SCHEMA, REPO_URL_SCHEMA,
     extract_repository_from_value,
-    get_encryption_key_password,
     record_repository,
     connect,
 };
@@ -52,7 +51,7 @@ pub async fn benchmark(
     let crypt_config = match keyfile {
         None => None,
         Some(path) => {
-            let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
+            let (key, _) = load_and_decrypt_key(&path, &crate::key::get_encryption_key_password)?;
             let crypt_config = CryptConfig::new(key)?;
             Some(Arc::new(crypt_config))
         }

src/bin/proxmox_backup_client/catalog.rs

@@ -0,0 +1,245 @@
+use std::os::unix::fs::OpenOptionsExt;
+use std::io::{Seek, SeekFrom};
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use anyhow::{bail, format_err, Error};
+use serde_json::Value;
+
+use proxmox::api::{api, cli::*};
+
+use proxmox_backup::tools;
+
+use proxmox_backup::client::*;
+
+use crate::{
+    REPO_URL_SCHEMA,
+    extract_repository_from_value,
+    record_repository,
+    load_and_decrypt_key,
+    api_datastore_latest_snapshot,
+    complete_repository,
+    complete_backup_snapshot,
+    complete_group_or_snapshot,
+    complete_pxar_archive_name,
+    connect,
+    BackupDir,
+    BackupGroup,
+    BufferedDynamicReader,
+    BufferedDynamicReadAt,
+    CatalogReader,
+    CATALOG_NAME,
+    CryptConfig,
+    DynamicIndexReader,
+    IndexFile,
+    Shell,
+};
+
+use crate::key::get_encryption_key_password;
+
+#[api(
+   input: {
+        properties: {
+            repository: {
+                schema: REPO_URL_SCHEMA,
+                optional: true,
+            },
+            snapshot: {
+                type: String,
+                description: "Snapshot path.",
+             },
+        }
+   }
+)]
+/// Dump catalog.
+async fn dump_catalog(param: Value) -> Result<Value, Error> {
+
+    let repo = extract_repository_from_value(&param)?;
+
+    let path = tools::required_string_param(&param, "snapshot")?;
+    let snapshot: BackupDir = path.parse()?;
+
+    let keyfile = param["keyfile"].as_str().map(PathBuf::from);
+
+    let crypt_config = match keyfile {
+        None => None,
+        Some(path) => {
+            let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
+            Some(Arc::new(CryptConfig::new(key)?))
+        }
+    };
+
+    let client = connect(repo.host(), repo.user())?;
+
+    let client = BackupReader::start(
+        client,
+        crypt_config.clone(),
+        repo.store(),
+        &snapshot.group().backup_type(),
+        &snapshot.group().backup_id(),
+        snapshot.backup_time(),
+        true,
+    ).await?;
+
+    let manifest = client.download_manifest().await?;
+
+    let index = client.download_dynamic_index(&manifest, CATALOG_NAME).await?;
+
+    let most_used = index.find_most_used_chunks(8);
+
+    let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
+
+    let mut reader = BufferedDynamicReader::new(index, chunk_reader);
+
+    let mut catalogfile = std::fs::OpenOptions::new()
+        .write(true)
+        .read(true)
+        .custom_flags(libc::O_TMPFILE)
+        .open("/tmp")?;
+
+    std::io::copy(&mut reader, &mut catalogfile)
+        .map_err(|err| format_err!("unable to download catalog - {}", err))?;
+
+    catalogfile.seek(SeekFrom::Start(0))?;
+
+    let mut catalog_reader = CatalogReader::new(catalogfile);
+
+    catalog_reader.dump()?;
+
+    record_repository(&repo);
+
+    Ok(Value::Null)
+}
+
+#[api(
+    input: {
+        properties: {
+            "snapshot": {
+                type: String,
+                description: "Group/Snapshot path.",
+            },
+            "archive-name": {
+                type: String,
+                description: "Backup archive name.",
+            },
+            "repository": {
+                optional: true,
+                schema: REPO_URL_SCHEMA,
+            },
+            "keyfile": {
+                optional: true,
+                type: String,
+                description: "Path to encryption key.",
+            },
+        },
+    },
+)]
+/// Shell to interactively inspect and restore snapshots.
+async fn catalog_shell(param: Value) -> Result<(), Error> {
+    let repo = extract_repository_from_value(&param)?;
+    let client = connect(repo.host(), repo.user())?;
+    let path = tools::required_string_param(&param, "snapshot")?;
+    let archive_name = tools::required_string_param(&param, "archive-name")?;
+
+    let (backup_type, backup_id, backup_time) = if path.matches('/').count() == 1 {
+        let group: BackupGroup = path.parse()?;
+        api_datastore_latest_snapshot(&client, repo.store(), group).await?
+    } else {
+        let snapshot: BackupDir = path.parse()?;
+        (snapshot.group().backup_type().to_owned(), snapshot.group().backup_id().to_owned(), snapshot.backup_time())
+    };
+
+    let keyfile = param["keyfile"].as_str().map(|p| PathBuf::from(p));
+    let crypt_config = match keyfile {
+        None => None,
+        Some(path) => {
+            let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
+            Some(Arc::new(CryptConfig::new(key)?))
+        }
+    };
+
+    let server_archive_name = if archive_name.ends_with(".pxar") {
+        format!("{}.didx", archive_name)
+    } else {
+        bail!("Can only mount pxar archives.");
+    };
+
+    let client = BackupReader::start(
+        client,
+        crypt_config.clone(),
+        repo.store(),
+        &backup_type,
+        &backup_id,
+        backup_time,
+        true,
+    ).await?;
+
+    let mut tmpfile = std::fs::OpenOptions::new()
+        .write(true)
+        .read(true)
+        .custom_flags(libc::O_TMPFILE)
+        .open("/tmp")?;
+
+    let manifest = client.download_manifest().await?;
+
+    let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
+    let most_used = index.find_most_used_chunks(8);
+    let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), most_used);
+    let reader = BufferedDynamicReader::new(index, chunk_reader);
+    let archive_size = reader.archive_size();
+    let reader: proxmox_backup::pxar::fuse::Reader =
+        Arc::new(BufferedDynamicReadAt::new(reader));
+    let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
+
+    client.download(CATALOG_NAME, &mut tmpfile).await?;
+    let index = DynamicIndexReader::new(tmpfile)
+        .map_err(|err| format_err!("unable to read catalog index - {}", err))?;
+
+    // Note: do not use values stored in index (not trusted) - instead, computed them again
+    let (csum, size) = index.compute_csum();
+    manifest.verify_file(CATALOG_NAME, &csum, size)?;
+
+    let most_used = index.find_most_used_chunks(8);
+    let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
+    let mut reader = BufferedDynamicReader::new(index, chunk_reader);
+    let mut catalogfile = std::fs::OpenOptions::new()
+        .write(true)
+        .read(true)
+        .custom_flags(libc::O_TMPFILE)
+        .open("/tmp")?;
+
+    std::io::copy(&mut reader, &mut catalogfile)
+        .map_err(|err| format_err!("unable to download catalog - {}", err))?;
+
+    catalogfile.seek(SeekFrom::Start(0))?;
+    let catalog_reader = CatalogReader::new(catalogfile);
+    let state = Shell::new(
+        catalog_reader,
+        &server_archive_name,
+        decoder,
+    ).await?;
+
+    println!("Starting interactive shell");
+    state.shell().await?;
+
+    record_repository(&repo);
+
+    Ok(())
+}
+
+pub fn catalog_mgmt_cli() -> CliCommandMap {
+    let catalog_shell_cmd_def = CliCommand::new(&API_METHOD_CATALOG_SHELL)
+        .arg_param(&["snapshot", "archive-name"])
+        .completion_cb("repository", complete_repository)
+        .completion_cb("archive-name", complete_pxar_archive_name)
+        .completion_cb("snapshot", complete_group_or_snapshot);
+
+    let catalog_dump_cmd_def = CliCommand::new(&API_METHOD_DUMP_CATALOG)
+        .arg_param(&["snapshot"])
+        .completion_cb("repository", complete_repository)
+        .completion_cb("snapshot", complete_backup_snapshot);
+
+    CliCommandMap::new()
+        .insert("dump", catalog_dump_cmd_def)
+        .insert("shell", catalog_shell_cmd_def)
+}

src/bin/proxmox_backup_client/key.rs

@@ -0,0 +1,277 @@
+use std::path::PathBuf;
+
+use anyhow::{bail, Error};
+use chrono::{Local, TimeZone};
+use serde::{Deserialize, Serialize};
+use xdg::BaseDirectories;
+
+use proxmox::api::api;
+use proxmox::api::cli::{CliCommand, CliCommandMap};
+use proxmox::sys::linux::tty;
+use proxmox::tools::fs::{file_get_contents, replace_file, CreateOptions};
+
+use proxmox_backup::backup::{
+    encrypt_key_with_passphrase, load_and_decrypt_key, store_key_config, KeyConfig,
+};
+use proxmox_backup::tools;
+
+pub fn master_pubkey_path() -> Result<PathBuf, Error> {
+    let base = BaseDirectories::with_prefix("proxmox-backup")?;
+
+    // usually $HOME/.config/proxmox-backup/master-public.pem
+    let path = base.place_config_file("master-public.pem")?;
+
+    Ok(path)
+}
+
+pub fn default_encryption_key_path() -> Result<PathBuf, Error> {
+    let base = BaseDirectories::with_prefix("proxmox-backup")?;
+
+    // usually $HOME/.config/proxmox-backup/encryption-key.json
+    let path = base.place_config_file("encryption-key.json")?;
+
+    Ok(path)
+}
+
+pub fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
+    // fixme: implement other input methods
+
+    use std::env::VarError::*;
+    match std::env::var("PBS_ENCRYPTION_PASSWORD") {
+        Ok(p) => return Ok(p.as_bytes().to_vec()),
+        Err(NotUnicode(_)) => bail!("PBS_ENCRYPTION_PASSWORD contains bad characters"),
+        Err(NotPresent) => {
+            // Try another method
+        }
+    }
+
+    // If we're on a TTY, query the user for a password
+    if tty::stdin_isatty() {
+        return Ok(tty::read_password("Encryption Key Password: ")?);
+    }
+
+    bail!("no password input mechanism available");
+}
+
+/// Convenience helper to get the default key file path only if it exists.
+pub fn optional_default_key_path() -> Result<Option<PathBuf>, Error> {
+    let path = default_encryption_key_path()?;
+    Ok(if path.exists() {
+        Some(path)
+    } else {
+        None
+    })
+}
+
+#[api(
+    default: "scrypt",
+)]
+#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+/// Key derivation function for password protected encryption keys.
+pub enum Kdf {
+    /// Do not encrypt the key.
+    None,
+
+    /// Encrypt they key with a password using SCrypt.
+    Scrypt,
+}
+
+impl Default for Kdf {
+    #[inline]
+    fn default() -> Self {
+        Kdf::Scrypt
+    }
+}
+
+#[api(
+    input: {
+        properties: {
+            kdf: {
+                type: Kdf,
+                optional: true,
+            },
+            path: {
+                description:
+                    "Output file. Without this the key will become the new default encryption key.",
+                optional: true,
+            }
+        },
+    },
+)]
+/// Create a new encryption key.
+fn create(kdf: Option<Kdf>, path: Option<String>) -> Result<(), Error> {
+    let path = match path {
+        Some(path) => PathBuf::from(path),
+        None => default_encryption_key_path()?,
+    };
+
+    let kdf = kdf.unwrap_or_default();
+
+    let key = proxmox::sys::linux::random_data(32)?;
+
+    match kdf {
+        Kdf::None => {
+            let created = Local.timestamp(Local::now().timestamp(), 0);
+
+            store_key_config(
+                &path,
+                false,
+                KeyConfig {
+                    kdf: None,
+                    created,
+                    modified: created,
+                    data: key,
+                },
+            )?;
+        }
+        Kdf::Scrypt => {
+            // always read passphrase from tty
+            if !tty::stdin_isatty() {
+                bail!("unable to read passphrase - no tty");
+            }
+
+            let password = tty::read_and_verify_password("Encryption Key Password: ")?;
+
+            let key_config = encrypt_key_with_passphrase(&key, &password)?;
+
+            store_key_config(&path, false, key_config)?;
+        }
+    }
+
+    Ok(())
+}
+
+#[api(
+    input: {
+        properties: {
+            kdf: {
+                type: Kdf,
+                optional: true,
+            },
+            path: {
+                description: "Key file. Without this the default key's password will be changed.",
+                optional: true,
+            }
+        },
+    },
+)]
+/// Change the encryption key's password.
+fn change_passphrase(kdf: Option<Kdf>, path: Option<String>) -> Result<(), Error> {
+    let path = match path {
+        Some(path) => PathBuf::from(path),
+        None => default_encryption_key_path()?,
+    };
+
+    let kdf = kdf.unwrap_or_default();
+
+    if !tty::stdin_isatty() {
+        bail!("unable to change passphrase - no tty");
+    }
+
+    let (key, created) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
+
+    match kdf {
+        Kdf::None => {
+            let modified = Local.timestamp(Local::now().timestamp(), 0);
+
+            store_key_config(
+                &path,
+                true,
+                KeyConfig {
+                    kdf: None,
+                    created, // keep original value
+                    modified,
+                    data: key.to_vec(),
+                },
+            )?;
+        }
+        Kdf::Scrypt => {
+            let password = tty::read_and_verify_password("New Password: ")?;
+
+            let mut new_key_config = encrypt_key_with_passphrase(&key, &password)?;
+            new_key_config.created = created; // keep original value
+
+            store_key_config(&path, true, new_key_config)?;
+        }
+    }
+
+    Ok(())
+}
+
+#[api(
+    input: {
+        properties: {
+            path: {
+                description: "Path to the PEM formatted RSA public key.",
+            },
+        },
+    },
+)]
+/// Import an RSA public key used to put an encrypted version of the symmetric backup encryption
+/// key onto the backup server along with each backup.
+fn import_master_pubkey(path: String) -> Result<(), Error> {
+    let pem_data = file_get_contents(&path)?;
+
+    if let Err(err) = openssl::pkey::PKey::public_key_from_pem(&pem_data) {
+        bail!("Unable to decode PEM data - {}", err);
+    }
+
+    let target_path = master_pubkey_path()?;
+
+    replace_file(&target_path, &pem_data, CreateOptions::new())?;
+
+    println!("Imported public master key to {:?}", target_path);
+
+    Ok(())
+}
+
+#[api]
+/// Create an RSA public/private key pair used to put an encrypted version of the symmetric backup
+/// encryption key onto the backup server along with each backup.
+fn create_master_key() -> Result<(), Error> {
+    // we need a TTY to query the new password
+    if !tty::stdin_isatty() {
+        bail!("unable to create master key - no tty");
+    }
+
+    let rsa = openssl::rsa::Rsa::generate(4096)?;
+    let pkey = openssl::pkey::PKey::from_rsa(rsa)?;
+
+    let password = String::from_utf8(tty::read_and_verify_password("Master Key Password: ")?)?;
+
+    let pub_key: Vec<u8> = pkey.public_key_to_pem()?;
+    let filename_pub = "master-public.pem";
+    println!("Writing public master key to {}", filename_pub);
+    replace_file(filename_pub, pub_key.as_slice(), CreateOptions::new())?;
+
+    let cipher = openssl::symm::Cipher::aes_256_cbc();
+    let priv_key: Vec<u8> = pkey.private_key_to_pem_pkcs8_passphrase(cipher, password.as_bytes())?;
+
+    let filename_priv = "master-private.pem";
+    println!("Writing private master key to {}", filename_priv);
+    replace_file(filename_priv, priv_key.as_slice(), CreateOptions::new())?;
+
+    Ok(())
+}
+
+pub fn cli() -> CliCommandMap {
+    let key_create_cmd_def = CliCommand::new(&API_METHOD_CREATE)
+        .arg_param(&["path"])
+        .completion_cb("path", tools::complete_file_name);
+
+    let key_change_passphrase_cmd_def = CliCommand::new(&API_METHOD_CHANGE_PASSPHRASE)
+        .arg_param(&["path"])
+        .completion_cb("path", tools::complete_file_name);
+
+    let key_create_master_key_cmd_def = CliCommand::new(&API_METHOD_CREATE_MASTER_KEY);
+    let key_import_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_IMPORT_MASTER_PUBKEY)
+        .arg_param(&["path"])
+        .completion_cb("path", tools::complete_file_name);
+
+    CliCommandMap::new()
+        .insert("create", key_create_cmd_def)
+        .insert("create-master-key", key_create_master_key_cmd_def)
+        .insert("import-master-pubkey", key_import_master_pubkey_cmd_def)
+        .insert("change-passphrase", key_change_passphrase_cmd_def)
+}

src/bin/proxmox_backup_client/mod.rs

@@ -4,3 +4,7 @@ mod mount;
 pub use mount::*;
 mod task;
 pub use task::*;
+mod catalog;
+pub use catalog::*;
+
+pub mod key;

src/bin/proxmox_backup_client/mount.rs

@@ -30,7 +30,6 @@ use proxmox_backup::client::*;
 use crate::{
     REPO_URL_SCHEMA,
     extract_repository_from_value,
-    get_encryption_key_password,
     complete_pxar_archive_name,
     complete_group_or_snapshot,
     complete_repository,
@@ -119,7 +118,7 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
     let crypt_config = match keyfile {
         None => None,
         Some(path) => {
-            let (key, _) = load_and_decrypt_key(&path, &get_encryption_key_password)?;
+            let (key, _) = load_and_decrypt_key(&path, &crate::key::get_encryption_key_password)?;
             Some(Arc::new(CryptConfig::new(key)?))
         }
     };

src/client/remote_chunk_reader.rs

@@ -10,11 +10,12 @@ use crate::backup::{AsyncReadChunk, CryptConfig, DataBlob, ReadChunk};
 use crate::tools::runtime::block_on;
 
 /// Read chunks from remote host using ``BackupReader``
+#[derive(Clone)]
 pub struct RemoteChunkReader {
     client: Arc<BackupReader>,
     crypt_config: Option<Arc<CryptConfig>>,
     cache_hint: HashMap<[u8; 32], usize>,
-    cache: Mutex<HashMap<[u8; 32], Vec<u8>>>,
+    cache: Arc<Mutex<HashMap<[u8; 32], Vec<u8>>>>,
 }
 
 impl RemoteChunkReader {
@@ -30,7 +31,7 @@ impl RemoteChunkReader {
             client,
             crypt_config,
             cache_hint,
-            cache: Mutex::new(HashMap::new()),
+            cache: Arc::new(Mutex::new(HashMap::new())),
         }
     }
 

src/tools/runtime.rs

@@ -56,30 +56,41 @@ extern {
 ///
 /// This makes sure that tokio's worker threads are marked for us so that we know whether we
 /// can/need to use `block_in_place` in our `block_on` helper.
-pub fn get_runtime() -> Arc<Runtime> {
+pub fn get_runtime_with_builder<F: Fn() -> runtime::Builder>(get_builder: F) -> Arc<Runtime> {
 
     let mut guard = RUNTIME.lock().unwrap();
 
     if let Some(rt) = guard.upgrade() { return rt; }
 
-    let rt = Arc::new(
+    let mut builder = get_builder();
-        runtime::Builder::new()
+    builder.on_thread_stop(|| {
-            .on_thread_stop(|| {
+        // avoid openssl bug: https://github.com/openssl/openssl/issues/6214
-                // avoid openssl bug: https://github.com/openssl/openssl/issues/6214
+        // call OPENSSL_thread_stop to avoid race with openssl cleanup handlers
-                // call OPENSSL_thread_stop to avoid race with openssl cleanup handlers
+        unsafe { OPENSSL_thread_stop(); }
-                unsafe { OPENSSL_thread_stop(); }
+    });
-            })
+
-            .threaded_scheduler()
+    let runtime = builder.build().expect("failed to spawn tokio runtime");
-            .enable_all()
+    let rt = Arc::new(runtime);
-            .build()
-            .expect("failed to spawn tokio runtime")
-    );
 
     *guard = Arc::downgrade(&rt.clone());
 
     rt
 }
 
+/// Get or create the current main tokio runtime.
+///
+/// This calls get_runtime_with_builder() using the tokio default threaded scheduler
+pub fn get_runtime() -> Arc<Runtime> {
+
+    get_runtime_with_builder(|| {
+        let mut builder = runtime::Builder::new();
+        builder.threaded_scheduler();
+        builder.enable_all();
+        builder
+    })
+}
+
+
 /// Block on a synchronous piece of code.
 pub fn block_in_place<R>(fut: impl FnOnce() -> R) -> R {
     // don't double-exit the context (tokio doesn't like that)

www/MainView.js

@@ -200,7 +200,13 @@ Ext.define('PBS.MainView', {
 		    xtype: 'versioninfo'
 		},
 		{
-		    flex: 1
+		    padding: 5,
+		    html: '<a href="https://bugzilla.proxmox.com" target="_blank">BETA</a>',
+		    baseCls: 'x-plain',
+		},
+		{
+		    flex: 1,
+		    baseCls: 'x-plain',
 		},
 		{
 		    baseCls: 'x-plain',

www/Makefile

@@ -19,6 +19,7 @@ JSSRC=							\
 	window/ACLEdit.js				\
 	window/DataStoreEdit.js				\
 	window/CreateDirectory.js			\
+	window/ZFSCreate.js				\
 	window/FileBrowser.js				\
 	window/BackupFileDownloader.js			\
 	dashboard/DataStoreStatistics.js		\

www/ZFSList.js

@@ -117,8 +117,7 @@ Ext.define('PBS.admin.ZFSList', {
 	    text: gettext('Detail'),
 	    xtype: 'proxmoxButton',
 	    disabled: true,
-	    handler: function() {
+	    handler: 'openDetailWindow',
-	    }
 	}
     ],
 

www/window/ZFSCreate.js

@@ -23,7 +23,8 @@ Ext.define('PBS.window.CreateZFS', {
 		    xtype: 'proxmoxtextfield',
 		    name: 'name',
 		    fieldLabel: gettext('Name'),
-		    allowBlank: false
+		    minLength: 3,
+		    allowBlank: false,
 		},
 		{
 		    xtype: 'proxmoxcheckbox',
@@ -75,7 +76,7 @@ Ext.define('PBS.window.CreateZFS', {
 		    xtype: 'pmxMultiDiskSelector',
 		    name: 'devices',
 		    nodename: 'localhost',
-		    typeParam: 'usage-type',
+		    typeParameter: 'usage-type',
 		    valueField: 'name',
 		    height: 200,
 		    emptyText: gettext('No Disks unused'),