bump version to 0.9.2-1

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "proxmox-backup"
-version = "0.9.0"
+version = "0.9.2"
 authors = ["Dietmar Maurer <dietmar@proxmox.com>"]
 edition = "2018"
 license = "AGPL-3"
@@ -29,7 +29,7 @@ hyper = "0.13.6"
 lazy_static = "1.4"
 libc = "0.2"
 log = "0.4"
-nix = "0.16"
+nix = "0.19"
 num-traits = "0.2"
 once_cell = "1.3.1"
 openssl = "0.10"
@@ -38,8 +38,8 @@ pam-sys = "0.5"
 percent-encoding = "2.1"
 pin-utils = "0.1.0"
 pathpatterns = "0.1.2"
-proxmox = { version = "0.4.2", features = [ "sortable-macro", "api-macro", "websocket" ] }
-#proxmox = { git = "ssh://gitolite3@proxdev.maurer-it.com/rust/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
+proxmox = { version = "0.5.0", features = [ "sortable-macro", "api-macro", "websocket" ] }
+#proxmox = { git = "git://git.proxmox.com/git/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
 #proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro", "websocket" ] }
 proxmox-fuse = "0.1.0"
 pxar = { version = "0.6.1", features = [ "tokio-io", "futures-io" ] }

README.rst

@@ -13,7 +13,7 @@ Versioning of proxmox helper crates
 
 To use current git master code of the proxmox* helper crates, add::
 
-   git = "ssh://gitolite3@proxdev.maurer-it.com/rust/proxmox"
+   git = "git://git.proxmox.com/git/proxmox"
 
 or::
 
@@ -22,6 +22,7 @@ or::
 to the proxmox dependency, and update the version to reflect the current,
 pre-release version number (e.g., "0.1.1-dev.1" instead of "0.1.0").
 
+
 Local cargo config
 ==================
 
@@ -35,3 +36,20 @@ checksums are not compatible.
 To reference new dependencies (or updated versions) that are not yet packaged,
 the dependency needs to point directly to a path or git source (e.g., see
 example for proxmox crate above).
+
+
+Build
+=====
+on Debian Buster
+
+Setup:
+  1. # echo 'deb http://download.proxmox.com/debian/devel/ buster main' >> /etc/apt/sources.list.d/proxmox-devel.list
+  2. # sudo wget http://download.proxmox.com/debian/proxmox-ve-release-6.x.gpg -O /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg
+  3. # sudo apt update
+  4. # sudo apt install devscripts debcargo clang
+  5. # git clone git://git.proxmox.com/git/proxmox-backup.git
+  6. # sudo mk-build-deps -ir
+
+Note: 2. may be skipped if you already added the PVE or PBS package repository
+
+You are now able to build using the Makefile or cargo itself.

debian/changelog

@@ -1,3 +1,137 @@
+rust-proxmox-backup (0.9.2-1) unstable; urgency=medium
+
+  * rework server web-interface, move more datastore related panels as tabs
+    inside the datastore view
+
+  * prune: never fail, just warn about failed removals
+
+  * prune/forget: skip snapshots with open readers (restore, verification)
+
+  * datastore: always ensure to remove individual snapshots before their group
+
+  * pxar: fix relative '!' rules in .pxarexclude
+
+  * pxar: anchor pxarexcludes starting with a slash
+
+  * GC: mark phase: ignore vanished index files
+
+  * server/rest: forward real client IP on proxied request and log it in
+    failed authentication requests
+
+  * server: rest: implement max URI path and query length request limits
+
+  * server/rest: implement request access log and log the query part of
+    URL and the user agent
+
+  * api: access: log to separate file, use syslog to errors only to reduce
+    syslog spam
+
+  * client: set HTTP connect timeout to 10 seconds
+
+  * client: sent TCP keep-alive after 2 minutes instead of the Linux default
+    of two hours.
+
+  * CLI completion: fix ACL path completion
+
+  * fix #2988: allow one to enable automatic verification after finishing a
+    snapshot, can be controlled as a per-datastore option
+
+  * various log-rotation improvements
+
+  * proxmox-backup-client: use HumanByte to render snapshot size
+
+  * paperkey: use svg as image format to provide better scalability
+
+  * backup: avoid Transport endpoint is not connected error
+
+  * fix #3038: check user before renewing ticket
+
+  * ui/tools: add zip module and allow to download an archive directory as a zip
+
+  * ui and api: add verification job config, allowing to schedule more
+    flexible jobs, filtering out already and/or recently verified snapshots
+    NOTE: the previous simple "verify all" schedule was dropped from the
+    datastore content, and does *not* gets migrated to the new job config.
+
+  * tasks: use systemd escape to decode/encode the task worker ID, avoiding
+    some display problems with problematic characters
+
+  * fix #2934: list also new to-be-installed packages in updates
+
+  * apt: add /changelog API call similar to PVE
+
+  * api: add world accessible ping dummy endpoint, to cheaply check for a
+    running PBS instance.
+
+  * ui: add datastore summary panel and move Statistics into it
+
+  * ui: navigation: add 'Add Datastore' button below datastore list
+
+  * ui: datastore panel: save and restore selected tab statefully
+
+  * send notification mails to email of root@pam account for GC and verify
+    jobs
+
+  * ui: datastore: use simple V. for verify action button
+
+  * ui: datastore: show snapshot manifest comment and allow to edit them
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 28 Oct 2020 21:27:02 +0100
+
+rust-proxmox-backup (0.9.1-1) unstable; urgency=medium
+
+  * TLS speedups (use SslAcceptor::mozilla_intermediate_v5)
+
+  * introduction.rst: add History
+
+  * fix #2847: proxmox-backup-client: add change-owner cmd
+
+  * proxmox-backup-client key: rename 'paper-key' command to 'paperkey'
+
+  * don't require WorkerTask in backup/ (introduce TaskState trait)
+
+  * fix #3070: replace internal with public URLs
+
+  * backup: index readers: drop useless shared lock
+
+  * add "Build" section to README.rst
+
+  * reader: actually allow users to download their own backups
+
+  * reader: track index chunks and limit access
+
+  * Userid: fix borrow/deref recursion
+
+  * depend on proxmox 0.4.3
+
+  * api: datastore: require allocate privilege for deletion
+
+  * fuse_loop: handle unmap on crashed instance
+
+  * fuse_loop: wait for instance to close after killing
+
+  * fuse_loop: add automatic cleanup of run files and dangling instances
+
+  * mount/map: use names for map/unmap for easier use
+
+  * ui: network: remove create VLAN option
+
+  * ui: Dashboard/TaskSummary: add Verifies to the Summary
+
+  * ui: implement task history limit and make it configurable
+
+  * docs: installation: add system requirements section
+
+  * client: implement map/unmap commands for .img backups
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 14 Oct 2020 13:42:12 +0200
+
+rust-proxmox-backup (0.9.0-2) unstable; urgency=medium
+
+  * ui: RemoteEdit: only send delete on update
+
+ -- Proxmox Support Team <support@proxmox.com>  Fri, 02 Oct 2020 15:37:45 +0200
+
 rust-proxmox-backup (0.9.0-1) unstable; urgency=medium
 
   * use ParallelHandler to verify chunks

debian/control

@@ -24,7 +24,7 @@ Build-Depends: debhelper (>= 11),
  librust-lazy-static-1+default-dev (>= 1.4-~~),
  librust-libc-0.2+default-dev,
  librust-log-0.4+default-dev,
- librust-nix-0.16+default-dev,
+ librust-nix-0.19+default-dev,
  librust-nom-5+default-dev (>= 5.1-~~),
  librust-num-traits-0.2+default-dev,
  librust-once-cell-1+default-dev (>= 1.3.1-~~),
@@ -34,10 +34,10 @@ Build-Depends: debhelper (>= 11),
  librust-pathpatterns-0.1+default-dev (>= 0.1.2-~~),
  librust-percent-encoding-2+default-dev (>= 2.1-~~),
  librust-pin-utils-0.1+default-dev,
- librust-proxmox-0.4+api-macro-dev (>= 0.4.2-~~),
- librust-proxmox-0.4+default-dev (>= 0.4.2-~~),
- librust-proxmox-0.4+sortable-macro-dev (>= 0.4.2-~~),
- librust-proxmox-0.4+websocket-dev (>= 0.4.2-~~),
+ librust-proxmox-0.5+api-macro-dev,
+ librust-proxmox-0.5+default-dev,
+ librust-proxmox-0.5+sortable-macro-dev,
+ librust-proxmox-0.5+websocket-dev,
  librust-proxmox-fuse-0.1+default-dev,
  librust-pxar-0.6+default-dev (>= 0.6.1-~~),
  librust-pxar-0.6+futures-io-dev (>= 0.6.1-~~),
@@ -107,7 +107,7 @@ Depends: fonts-font-awesome,
          pbs-i18n,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
-         proxmox-widget-toolkit (>= 2.2-4),
+         proxmox-widget-toolkit (>= 2.3-6),
          pve-xtermjs (>= 4.7.0-1),
          smartmontools,
          ${misc:Depends},
@@ -119,7 +119,9 @@ Description: Proxmox Backup Server daemon with tools and GUI
 
 Package: proxmox-backup-client
 Architecture: any
-Depends: qrencode ${misc:Depends}, ${shlibs:Depends}
+Depends: qrencode,
+         ${misc:Depends},
+         ${shlibs:Depends},
 Description: Proxmox Backup Client tools
  This package contains the Proxmox Backup client, which provides a
  simple command line tool to create and restore backups.

debian/control.in

@@ -7,7 +7,7 @@ Depends: fonts-font-awesome,
          pbs-i18n,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
-         proxmox-widget-toolkit (>= 2.3-1),
+         proxmox-widget-toolkit (>= 2.3-6),
          pve-xtermjs (>= 4.7.0-1),
          smartmontools,
          ${misc:Depends},
@@ -19,7 +19,9 @@ Description: Proxmox Backup Server daemon with tools and GUI
 
 Package: proxmox-backup-client
 Architecture: any
-Depends: qrencode ${misc:Depends}, ${shlibs:Depends}
+Depends: qrencode,
+         ${misc:Depends},
+         ${shlibs:Depends},
 Description: Proxmox Backup Client tools
  This package contains the Proxmox Backup client, which provides a
  simple command line tool to create and restore backups.

debian/postinst

@@ -15,6 +15,8 @@ case "$1" in
 	fi
 	deb-systemd-invoke $_dh_action proxmox-backup.service proxmox-backup-proxy.service >/dev/null || true
 
+	flock -w 30 /etc/proxmox-backup/.datastore.lck sed -i '/^\s\+verify-schedule /d' /etc/proxmox-backup/datastore.cfg
+
 	# FIXME: Remove in future version once we're sure no broken entries remain in anyone's files
 	if grep -q -e ':termproxy::[^@]\+: ' /var/log/proxmox-backup/tasks/active; then
 	    echo "Fixing up termproxy user id in task log..."

docs/_ext/proxmox-scanrefs.py

@@ -44,12 +44,13 @@ def scan_extjs_files(wwwdir="../www"): # a bit rough i know, but we can optimize
                 js_files.append(os.path.join(root, filename))
     for js_file in js_files:
         fd = open(js_file).read()
-        match = re.search("onlineHelp:\s*[\'\"](.*?)[\'\"]", fd) # match object is tuple
-        if match:
-            anchor = match.groups()[0]
+        allmatch = re.findall("onlineHelp:\s*[\'\"](.*?)[\'\"]", fd, re.M)
+        for match in allmatch:
+            anchor = match
             anchor = re.sub('_', '-', anchor) # normalize labels
             logger.info("found onlineHelp: {} in {}".format(anchor, js_file))
             used_anchors.append(anchor)
+
     return used_anchors
 
 

docs/backup-client.rst

@@ -0,0 +1,714 @@
+Backup Client Usage
+===================
+
+The command line client is called :command:`proxmox-backup-client`.
+
+
+Repository Locations
+--------------------
+
+The client uses the following notation to specify a datastore repository
+on the backup server.
+
+  [[username@]server[:port]:]datastore
+
+The default value for ``username`` is ``root@pam``.  If no server is specified,
+the default is the local host (``localhost``).
+
+You can specify a port if your backup server is only reachable on a different
+port (e.g. with NAT and port forwarding).
+
+Note that if the server is an IPv6 address, you have to write it with
+square brackets (e.g. [fe80::01]).
+
+You can pass the repository with the ``--repository`` command
+line option, or by setting the ``PBS_REPOSITORY`` environment
+variable.
+
+Here some examples of valid repositories and the real values
+
+================================ ============ ================== ===========
+Example                          User         Host:Port          Datastore
+================================ ============ ================== ===========
+mydatastore                      ``root@pam`` localhost:8007     mydatastore
+myhostname:mydatastore           ``root@pam`` myhostname:8007    mydatastore
+user@pbs@myhostname:mydatastore  ``user@pbs`` myhostname:8007    mydatastore
+192.168.55.55:1234:mydatastore   ``root@pam`` 192.168.55.55:1234 mydatastore
+[ff80::51]:mydatastore           ``root@pam`` [ff80::51]:8007    mydatastore
+[ff80::51]:1234:mydatastore      ``root@pam`` [ff80::51]:1234    mydatastore
+================================ ============ ================== ===========
+
+Environment Variables
+---------------------
+
+``PBS_REPOSITORY``
+  The default backup repository.
+
+``PBS_PASSWORD``
+  When set, this value is used for the password required for the
+  backup server.
+
+``PBS_ENCRYPTION_PASSWORD``
+  When set, this value is used to access the secret encryption key (if
+  protected by password).
+
+``PBS_FINGERPRINT`` When set, this value is used to verify the server
+  certificate (only used if the system CA certificates cannot
+  validate the certificate).
+
+
+Output Format
+-------------
+
+Most commands support the ``--output-format`` parameter. It accepts
+the following values:
+
+:``text``: Text format (default). Structured data is rendered as a table.
+
+:``json``: JSON (single line).
+
+:``json-pretty``: JSON (multiple lines, nicely formatted).
+
+
+Please use the following environment variables to modify output behavior:
+
+``PROXMOX_OUTPUT_FORMAT``
+  Defines the default output format.
+
+``PROXMOX_OUTPUT_NO_BORDER``
+  If set (to any value), do not render table borders.
+
+``PROXMOX_OUTPUT_NO_HEADER``
+  If set (to any value), do not render table headers.
+
+.. note:: The ``text`` format is designed to be human readable, and
+   not meant to be parsed by automation tools. Please use the ``json``
+   format if you need to process the output.
+
+
+.. _creating-backups:
+
+Creating Backups
+----------------
+
+This section explains how to create a backup from within the machine. This can
+be a physical host, a virtual machine, or a container. Such backups may contain file
+and image archives. There are no restrictions in this case.
+
+.. note:: If you want to backup virtual machines or containers on Proxmox VE, see :ref:`pve-integration`.
+
+For the following example you need to have a backup server set up, working
+credentials and need to know the repository name.
+In the following examples we use ``backup-server:store1``.
+
+.. code-block:: console
+
+  # proxmox-backup-client backup root.pxar:/ --repository backup-server:store1
+  Starting backup: host/elsa/2019-12-03T09:35:01Z
+  Client name: elsa
+  skip mount point: "/boot/efi"
+  skip mount point: "/dev"
+  skip mount point: "/run"
+  skip mount point: "/sys"
+  Uploaded 12129 chunks in 87 seconds (564 MB/s).
+  End Time: 2019-12-03T10:36:29+01:00
+
+This will prompt you for a password and then uploads a file archive named
+``root.pxar`` containing all the files in the ``/`` directory.
+
+.. Caution:: Please note that the proxmox-backup-client does not
+   automatically include mount points. Instead, you will see a short
+   ``skip mount point`` notice for each of them. The idea is to
+   create a separate file archive for each mounted disk. You can
+   explicitly include them using the ``--include-dev`` option
+   (i.e. ``--include-dev /boot/efi``). You can use this option
+   multiple times for each mount point that should be included.
+
+The ``--repository`` option can get quite long and is used by all
+commands. You can avoid having to enter this value by setting the
+environment variable ``PBS_REPOSITORY``. Note that if you would like this to remain set
+over multiple sessions, you should instead add the below line to your
+``.bashrc`` file.
+
+.. code-block:: console
+
+  # export PBS_REPOSITORY=backup-server:store1
+
+After this you can execute all commands without specifying the ``--repository``
+option.
+
+One single backup is allowed to contain more than one archive. For example, if
+you want to backup two disks mounted at ``/mnt/disk1`` and ``/mnt/disk2``:
+
+.. code-block:: console
+
+  # proxmox-backup-client backup disk1.pxar:/mnt/disk1 disk2.pxar:/mnt/disk2
+
+This creates a backup of both disks.
+
+The backup command takes a list of backup specifications, which
+include the archive name on the server, the type of the archive, and the
+archive source at the client. The format is:
+
+    <archive-name>.<type>:<source-path>
+
+Common types are ``.pxar`` for file archives, and ``.img`` for block
+device images. To create a backup of a block device run the following command:
+
+.. code-block:: console
+
+  # proxmox-backup-client backup mydata.img:/dev/mylvm/mydata
+
+
+Excluding files/folders from a backup
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes it is desired to exclude certain files or folders from a backup archive.
+To tell the Proxmox Backup client when and how to ignore files and directories,
+place a text file called ``.pxarexclude`` in the filesystem hierarchy.
+Whenever the backup client encounters such a file in a directory, it interprets
+each line as glob match patterns for files and directories that are to be excluded
+from the backup.
+
+The file must contain a single glob pattern per line. Empty lines are ignored.
+The same is true for lines starting with ``#``, which indicates a comment.
+A ``!`` at the beginning of a line reverses the glob match pattern from an exclusion
+to an explicit inclusion. This makes it possible to exclude all entries in a
+directory except for a few single files/subdirectories.
+Lines ending in ``/`` match only on directories.
+The directory containing the ``.pxarexclude`` file is considered to be the root of
+the given patterns. It is only possible to match files in this directory and its subdirectories.
+
+``\`` is used to escape special glob characters.
+``?`` matches any single character.
+``*`` matches any character, including an empty string.
+``**`` is used to match subdirectories. It can be used to, for example, exclude
+all files ending in ``.tmp`` within the directory or subdirectories with the
+following pattern ``**/*.tmp``.
+``[...]`` matches a single character from any of the provided characters within
+the brackets. ``[!...]`` does the complementary and matches any single character
+not contained within the brackets. It is also possible to specify ranges with two
+characters separated by ``-``. For example, ``[a-z]`` matches any lowercase
+alphabetic character and ``[0-9]`` matches any one single digit.
+
+The order of the glob match patterns defines whether a file is included or
+excluded, that is to say later entries override previous ones.
+This is also true for match patterns encountered deeper down the directory tree,
+which can override a previous exclusion.
+Be aware that excluded directories will **not** be read by the backup client.
+Thus, a ``.pxarexclude`` file in an excluded subdirectory will have no effect.
+``.pxarexclude`` files are treated as regular files and will be included in the
+backup archive.
+
+For example, consider the following directory structure:
+
+.. code-block:: console
+
+    # ls -aR folder
+    folder/:
+    .  ..  .pxarexclude  subfolder0  subfolder1
+
+    folder/subfolder0:
+    .  ..  file0  file1  file2  file3  .pxarexclude
+
+    folder/subfolder1:
+    .  ..  file0  file1  file2  file3
+
+The different ``.pxarexclude`` files contain the following:
+
+.. code-block:: console
+
+    # cat folder/.pxarexclude
+    /subfolder0/file1
+    /subfolder1/*
+    !/subfolder1/file2
+
+.. code-block:: console
+
+    # cat folder/subfolder0/.pxarexclude
+    file3
+
+This would exclude ``file1`` and ``file3`` in ``subfolder0`` and all of
+``subfolder1`` except ``file2``.
+
+Restoring this backup will result in:
+
+.. code-block:: console
+
+    ls -aR restored
+    restored/:
+    .  ..  .pxarexclude  subfolder0  subfolder1
+
+    restored/subfolder0:
+    .  ..  file0  file2  .pxarexclude
+
+    restored/subfolder1:
+    .  ..  file2
+
+
+.. _encryption:
+
+Encryption
+----------
+
+Proxmox Backup supports client-side encryption with AES-256 in GCM_
+mode. To set this up, you first need to create an encryption key:
+
+.. code-block:: console
+
+  # proxmox-backup-client key create my-backup.key
+  Encryption Key Password: **************
+
+The key is password protected by default. If you do not need this
+extra protection, you can also create it without a password:
+
+.. code-block:: console
+
+  # proxmox-backup-client key create /path/to/my-backup.key --kdf none
+
+Having created this key, it is now possible to create an encrypted backup, by
+passing the ``--keyfile`` parameter, with the path to the key file.
+
+.. code-block:: console
+
+  # proxmox-backup-client backup etc.pxar:/etc --keyfile /path/to/my-backup.key
+  Password: *********
+  Encryption Key Password: **************
+  ...
+
+.. Note:: If you do not specify the name of the backup key, the key will be
+  created in the default location
+  ``~/.config/proxmox-backup/encryption-key.json``. ``proxmox-backup-client``
+  will also search this location by default, in case the ``--keyfile``
+  parameter is not specified.
+
+You can avoid entering the passwords by setting the environment
+variables ``PBS_PASSWORD`` and ``PBS_ENCRYPTION_PASSWORD``.
+
+
+Using a master key to store and recover encryption keys
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can also use ``proxmox-backup-client key`` to create an RSA public/private
+key pair, which can be used to store an encrypted version of the symmetric
+backup encryption key alongside each backup and recover it later.
+
+To set up a master key:
+
+1. Create an encryption key for the backup:
+
+   .. code-block:: console
+
+     # proxmox-backup-client key create
+     creating default key at: "~/.config/proxmox-backup/encryption-key.json"
+     Encryption Key Password: **********
+     ...
+
+   The resulting file will be saved to ``~/.config/proxmox-backup/encryption-key.json``.
+
+2. Create an RSA public/private key pair:
+
+   .. code-block:: console
+
+     # proxmox-backup-client key create-master-key
+     Master Key Password: *********
+     ...
+
+   This will create two files in your current directory, ``master-public.pem``
+   and ``master-private.pem``.
+
+3. Import the newly created ``master-public.pem`` public certificate, so that
+   ``proxmox-backup-client`` can find and use it upon backup.
+
+   .. code-block:: console
+
+     # proxmox-backup-client key import-master-pubkey /path/to/master-public.pem
+     Imported public master key to "~/.config/proxmox-backup/master-public.pem"
+
+4. With all these files in place, run a backup job:
+
+   .. code-block:: console
+
+     # proxmox-backup-client backup etc.pxar:/etc
+
+   The key will be stored in your backup, under the name ``rsa-encrypted.key``.
+
+   .. Note:: The ``--keyfile`` parameter can be excluded, if the encryption key
+     is in the default path. If you specified another path upon creation, you
+     must pass the ``--keyfile`` parameter.
+
+5. To test that everything worked, you can restore the key from the backup:
+
+   .. code-block:: console
+
+     # proxmox-backup-client restore /path/to/backup/ rsa-encrypted.key /path/to/target
+
+   .. Note:: You should not need an encryption key to extract this file. However, if
+     a key exists at the default location
+     (``~/.config/proxmox-backup/encryption-key.json``) the program will prompt
+     you for an encryption key password. Simply moving ``encryption-key.json``
+     out of this directory will fix this issue.
+
+6. Then, use the previously generated master key to decrypt the file:
+
+   .. code-block:: console
+
+     # openssl rsautl -decrypt -inkey master-private.pem -in rsa-encrypted.key -out /path/to/target
+     Enter pass phrase for ./master-private.pem: *********
+
+7. The target file will now contain the encryption key information in plain
+   text. The success of this can be confirmed by passing the resulting ``json``
+   file, with the ``--keyfile`` parameter, when decrypting files from the backup.
+
+.. warning:: Without their key, backed up files will be inaccessible. Thus, you should
+  keep keys ordered and in a place that is separate from the contents being
+  backed up. It can happen, for example, that you back up an entire system, using
+  a key on that system. If the system then becomes inaccessible for any reason
+  and needs to be restored, this will not be possible as the encryption key will be
+  lost along with the broken system. In preparation for the worst case scenario,
+  you should consider keeping a paper copy of this key locked away in
+  a safe place.
+
+
+Restoring Data
+--------------
+
+The regular creation of backups is a necessary step to avoiding data
+loss. More importantly, however, is the restoration. It is good practice to perform
+periodic recovery tests to ensure that you can access the data in
+case of problems.
+
+First, you need to find the snapshot which you want to restore. The snapshot
+command provides a list of all the snapshots on the server:
+
+.. code-block:: console
+
+  # proxmox-backup-client snapshots
+  ┌────────────────────────────────┬─────────────┬────────────────────────────────────┐
+  │ snapshot                       │        size │ files                              │
+  ╞════════════════════════════════╪═════════════╪════════════════════════════════════╡
+  │ host/elsa/2019-12-03T09:30:15Z │ 51788646825 │ root.pxar catalog.pcat1 index.json │
+  ├────────────────────────────────┼─────────────┼────────────────────────────────────┤
+  │ host/elsa/2019-12-03T09:35:01Z │ 51790622048 │ root.pxar catalog.pcat1 index.json │
+  ├────────────────────────────────┼─────────────┼────────────────────────────────────┤
+  ...
+
+You can inspect the catalog to find specific files.
+
+.. code-block:: console
+
+  # proxmox-backup-client catalog dump host/elsa/2019-12-03T09:35:01Z
+  ...
+  d "./root.pxar.didx/etc/cifs-utils"
+  l "./root.pxar.didx/etc/cifs-utils/idmap-plugin"
+  d "./root.pxar.didx/etc/console-setup"
+  ...
+
+The restore command lets you restore a single archive from the
+backup.
+
+.. code-block:: console
+
+  # proxmox-backup-client restore host/elsa/2019-12-03T09:35:01Z root.pxar /target/path/
+
+To get the contents of any archive, you can restore the ``index.json`` file in the
+repository to the target path '-'. This will dump the contents to the standard output.
+
+.. code-block:: console
+
+  # proxmox-backup-client restore host/elsa/2019-12-03T09:35:01Z index.json -
+
+
+Interactive Restores
+~~~~~~~~~~~~~~~~~~~~
+
+If you only want to restore a few individual files, it is often easier
+to use the interactive recovery shell.
+
+.. code-block:: console
+
+  # proxmox-backup-client catalog shell host/elsa/2019-12-03T09:35:01Z root.pxar
+  Starting interactive shell
+  pxar:/ > ls
+  bin        boot       dev        etc        home       lib        lib32
+  ...
+
+The interactive recovery shell is a minimal command line interface that
+utilizes the metadata stored in the catalog to quickly list, navigate and
+search files in a file archive.
+To restore files, you can select them individually or match them with a glob
+pattern.
+
+Using the catalog for navigation reduces the overhead considerably because only
+the catalog needs to be downloaded and, optionally, decrypted.
+The actual chunks are only accessed if the metadata in the catalog is not enough
+or for the actual restore.
+
+Similar to common UNIX shells ``cd`` and ``ls`` are the commands used to change
+working directory and list directory contents in the archive.
+``pwd`` shows the full path of the current working directory with respect to the
+archive root.
+
+Being able to quickly search the contents of the archive is a commonly needed feature.
+That's where the catalog is most valuable.
+For example:
+
+.. code-block:: console
+
+  pxar:/ > find etc/**/*.txt --select
+  "/etc/X11/rgb.txt"
+  pxar:/ > list-selected
+  etc/**/*.txt
+  pxar:/ > restore-selected /target/path
+  ...
+
+This will find and print all files ending in ``.txt`` located in ``etc/`` or a
+subdirectory and add the corresponding pattern to the list for subsequent restores.
+``list-selected`` shows these patterns and ``restore-selected`` finally restores
+all files in the archive matching the patterns to ``/target/path`` on the local
+host. This will scan the whole archive.
+
+With ``restore /target/path`` you can restore the sub-archive given by the current
+working directory to the local target path ``/target/path`` on your host.
+By additionally passing a glob pattern with ``--pattern <glob>``, the restore is
+further limited to files matching the pattern.
+For example:
+
+.. code-block:: console
+
+  pxar:/ > cd /etc/
+  pxar:/etc/ > restore /target/ --pattern **/*.conf
+  ...
+
+The above will scan trough all the directories below ``/etc`` and restore all
+files ending in ``.conf``.
+
+.. todo:: Explain interactive restore in more detail
+
+Mounting of Archives via FUSE
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :term:`FUSE` implementation for the pxar archive allows you to mount a
+file archive as a read-only filesystem to a mountpoint on your host.
+
+.. code-block:: console
+
+  # proxmox-backup-client mount host/backup-client/2020-01-29T11:29:22Z root.pxar /mnt/mountpoint
+  # ls /mnt/mountpoint
+  bin   dev  home  lib32  libx32      media  opt   root  sbin  sys  usr
+  boot  etc  lib   lib64  lost+found  mnt    proc  run   srv   tmp  var
+
+This allows you to access the full contents of the archive in a seamless manner.
+
+.. note:: As the FUSE connection needs to fetch and decrypt chunks from the
+    backup server's datastore, this can cause some additional network and CPU
+    load on your host, depending on the operations you perform on the mounted
+    filesystem.
+
+To unmount the filesystem use the ``umount`` command on the mountpoint:
+
+.. code-block:: console
+
+  # umount /mnt/mountpoint
+
+Login and Logout
+----------------
+
+The client tool prompts you to enter the logon password as soon as you
+want to access the backup server. The server checks your credentials
+and responds with a ticket that is valid for two hours. The client
+tool automatically stores that ticket and uses it for further requests
+to this server.
+
+You can also manually trigger this login/logout using the login and
+logout commands:
+
+.. code-block:: console
+
+  # proxmox-backup-client login
+  Password: **********
+
+To remove the ticket, issue a logout:
+
+.. code-block:: console
+
+  # proxmox-backup-client logout
+
+
+.. _backup-pruning:
+
+Pruning and Removing Backups
+----------------------------
+
+You can manually delete a backup snapshot using the ``forget``
+command:
+
+.. code-block:: console
+
+  # proxmox-backup-client forget <snapshot>
+
+
+.. caution:: This command removes all archives in this backup
+   snapshot. They will be inaccessible and unrecoverable.
+
+
+Although manual removal is sometimes required, the ``prune``
+command is normally used to systematically delete older backups. Prune lets
+you specify which backup snapshots you want to keep. The
+following retention options are available:
+
+``--keep-last <N>``
+  Keep the last ``<N>`` backup snapshots.
+
+``--keep-hourly <N>``
+  Keep backups for the last ``<N>`` hours. If there is more than one
+  backup for a single hour, only the latest is kept.
+
+``--keep-daily <N>``
+  Keep backups for the last ``<N>`` days. If there is more than one
+  backup for a single day, only the latest is kept.
+
+``--keep-weekly <N>``
+  Keep backups for the last ``<N>`` weeks. If there is more than one
+  backup for a single week, only the latest is kept.
+
+  .. note:: Weeks start on Monday and end on Sunday. The software
+     uses the `ISO week date`_ system and handles weeks at
+     the end of the year correctly.
+
+``--keep-monthly <N>``
+  Keep backups for the last ``<N>`` months. If there is more than one
+  backup for a single month, only the latest is kept.
+
+``--keep-yearly <N>``
+  Keep backups for the last ``<N>`` years. If there is more than one
+  backup for a single year, only the latest is kept.
+
+The retention options are processed in the order given above. Each option
+only covers backups within its time period. The next option does not take care
+of already covered backups. It will only consider older backups.
+
+Unfinished and incomplete backups will be removed by the prune command unless
+they are newer than the last successful backup. In this case, the last failed
+backup is retained.
+
+.. code-block:: console
+
+  # proxmox-backup-client prune <group> --keep-daily 7 --keep-weekly 4 --keep-monthly 3
+
+
+You can use the ``--dry-run`` option to test your settings. This only
+shows the list of existing snapshots and what actions prune would take.
+
+.. code-block:: console
+
+  # proxmox-backup-client prune host/elsa --dry-run --keep-daily 1 --keep-weekly 3
+  ┌────────────────────────────────┬──────┐
+  │ snapshot                       │ keep │
+  ╞════════════════════════════════╪══════╡
+  │ host/elsa/2019-12-04T13:20:37Z │    1 │
+  ├────────────────────────────────┼──────┤
+  │ host/elsa/2019-12-03T09:35:01Z │    0 │
+  ├────────────────────────────────┼──────┤
+  │ host/elsa/2019-11-22T11:54:47Z │    1 │
+  ├────────────────────────────────┼──────┤
+  │ host/elsa/2019-11-21T12:36:25Z │    0 │
+  ├────────────────────────────────┼──────┤
+  │ host/elsa/2019-11-10T10:42:20Z │    1 │
+  └────────────────────────────────┴──────┘
+
+.. note:: Neither the ``prune`` command nor the ``forget`` command free space
+   in the chunk-store. The chunk-store still contains the data blocks. To free
+   space you need to perform :ref:`garbage-collection`.
+
+
+.. _garbage-collection:
+
+Garbage Collection
+------------------
+
+The ``prune`` command removes only the backup index files, not the data
+from the datastore. This task is left to the garbage collection
+command. It is recommended to carry out garbage collection on a regular basis.
+
+The garbage collection works in two phases. In the first phase, all
+data blocks that are still in use are marked. In the second phase,
+unused data blocks are removed.
+
+.. note:: This command needs to read all existing backup index files
+  and touches the complete chunk-store. This can take a long time
+  depending on the number of chunks and the speed of the underlying
+  disks.
+
+.. note:: The garbage collection will only remove chunks that haven't been used
+   for at least one day (exactly 24h 5m). This grace period is necessary because
+   chunks in use are marked by touching the chunk which updates the ``atime``
+   (access time) property. Filesystems are mounted with the ``relatime`` option
+   by default. This results in a better performance by only updating the
+   ``atime`` property if the last access has been at least 24 hours ago. The
+   downside is, that touching a chunk within these 24 hours will not always
+   update its ``atime`` property.
+
+   Chunks in the grace period will be logged at the end of the garbage
+   collection task as *Pending removals*.
+
+.. code-block:: console
+
+  # proxmox-backup-client garbage-collect
+  starting garbage collection on store store2
+  Start GC phase1 (mark used chunks)
+  Start GC phase2 (sweep unused chunks)
+  percentage done: 1, chunk count: 219
+  percentage done: 2, chunk count: 453
+  ...
+  percentage done: 99, chunk count: 21188
+  Removed bytes: 411368505
+  Removed chunks: 203
+  Original data bytes: 327160886391
+  Disk bytes: 52767414743 (16 %)
+  Disk chunks: 21221
+  Average chunk size: 2486565
+  TASK OK
+
+
+.. todo:: howto run garbage-collection at regular intervals (cron)
+
+Benchmarking
+------------
+
+The backup client also comes with a benchmarking tool. This tool measures
+various metrics relating to compression and encryption speeds. You can run a
+benchmark using the ``benchmark`` subcommand of ``proxmox-backup-client``:
+
+.. code-block:: console
+
+  # proxmox-backup-client benchmark
+  Uploaded 656 chunks in 5 seconds.
+  Time per request: 7659 microseconds.
+  TLS speed: 547.60 MB/s
+  SHA256 speed: 585.76 MB/s
+  Compression speed: 1923.96 MB/s
+  Decompress speed: 7885.24 MB/s
+  AES256/GCM speed: 3974.03 MB/s
+  ┌───────────────────────────────────┬─────────────────────┐
+  │ Name                              │ Value               │
+  ╞═══════════════════════════════════╪═════════════════════╡
+  │ TLS (maximal backup upload speed) │ 547.60 MB/s (93%)   │
+  ├───────────────────────────────────┼─────────────────────┤
+  │ SHA256 checksum computation speed │ 585.76 MB/s (28%)   │
+  ├───────────────────────────────────┼─────────────────────┤
+  │ ZStd level 1 compression speed    │ 1923.96 MB/s (89%)  │
+  ├───────────────────────────────────┼─────────────────────┤
+  │ ZStd level 1 decompression speed  │ 7885.24 MB/s (98%)  │
+  ├───────────────────────────────────┼─────────────────────┤
+  │ AES256 GCM encryption speed       │ 3974.03 MB/s (104%) │
+  └───────────────────────────────────┴─────────────────────┘
+
+.. note:: The percentages given in the output table correspond to a
+  comparison against a Ryzen 7 2700X. The TLS test connects to the
+  local host, so there is no network involved.
+
+You can also pass the ``--output-format`` parameter to output stats in ``json``,
+rather than the default table format.
+
+

docs/calendarevents.rst

@@ -13,7 +13,7 @@ by the systemd Time and Date Specification (see `systemd.time manpage`_)
 called `calendar events` for its schedules.
 
 `Calendar events` are expressions to specify one or more points in time.
-They are mostly compatible with systemds calendar events.
+They are mostly compatible with systemd's calendar events.
 
 The general format is as follows:
 
@@ -27,7 +27,7 @@ If the weekday or date part is omitted, all (week)days are included.
 If the time part is omitted, the time 00:00:00 is implied.
 (e.g. '2020-01-01' refers to '2020-01-01 00:00:00')
 
-Weekdays are specified with the abbreviated english version:
+Weekdays are specified with the abbreviated English version:
 `mon, tue, wed, thu, fri, sat, sun`.
 
 Each field can contain multiple values in the following formats:
@@ -48,7 +48,7 @@ Value                              Syntax
 `daily`                            `*-*-* 00:00:00`
 `weekly`                           `mon *-*-* 00:00:00`
 `monthly`                          `*-*-01 00:00:00`
-`yearly` or `annualy`              `*-01-01 00:00:00`
+`yearly` or `annually`              `*-01-01 00:00:00`
 `quarterly`                        `*-01,04,07,10-01 00:00:00`
 `semiannually` or `semi-annually`  `*-01,07-01 00:00:00`
 =================================  ==============================
@@ -80,7 +80,7 @@ Differences to systemd
 
 Not all features of systemd calendar events are implemented:
 
-* no unix timestamps (e.g. `@12345`): instead use date and time to specify
+* no Unix timestamps (e.g. `@12345`): instead use date and time to specify
   a specific point in time
 * no timezone: all schedules use the set timezone on the server
 * no sub-second resolution

docs/command-line-tools.rst

@@ -12,7 +12,7 @@ Command Line Tools
 .. include:: proxmox-backup-manager/description.rst
 
 ``pxar``
-~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. include:: pxar/description.rst
 

docs/command-syntax.rst

@@ -10,7 +10,7 @@ Command Syntax
 Catalog Shell Commands
 ~~~~~~~~~~~~~~~~~~~~~~
 
-Those command are available when you start an intercative restore shell:
+Those command are available when you start an interactive restore shell:
 
 .. code-block:: console
 

docs/epilog.rst

@@ -1,6 +1,6 @@
 .. Epilog (included at top of each file)
    
-   We use this file to define external links and commone replacement
+   We use this file to define external links and common replacement
    patterns.
 
 .. |VERSION| replace:: 1.0

docs/faq.rst

@@ -34,7 +34,7 @@ How long will my Proxmox Backup Server version be supported?
 Can I copy or synchronize my datastore to another location?
 -----------------------------------------------------------
 
-Proxmox Backup Server allows you to copy or synchroize datastores to other
+Proxmox Backup Server allows you to copy or synchronize datastores to other
 locations, through the use of *Remotes* and *Sync Jobs*. *Remote* is the term
 given to a separate server, which has a datastore that can be synced to a local store.
 A *Sync Job* is the process which is used to pull the contents of a datastore from

docs/gui.rst

@@ -0,0 +1,135 @@
+Graphical User Interface
+========================
+
+Proxmox Backup Server offers an integrated, web-based interface to manage the
+server. This means that you can carry out all administration tasks through your
+web browser, and that you don't have to worry about installing extra management
+tools. The web interface also provides a built in console, so if you prefer the
+command line or need some extra control, you have this option.
+
+The web interface can be accessed via https://youripaddress:8007. The default
+login is `root`, and the password is the one specified during the installation
+process.
+
+
+Features
+--------
+
+* Simple management interface for Proxmox Backup Server
+* Monitoring of tasks, logs and resource usage
+* Management of users, permissions, datastores, etc.
+* Secure HTML5 console
+* Support for multiple authentication sources
+* Support for multiple languages
+* Based on ExtJS 6.x JavaScript framework
+
+
+Login
+-----
+
+.. image:: images/screenshots/pbs-gui-login-window.png
+  :width: 250
+  :align: right
+  :alt: PBS login window
+
+When you connect to the web interface, you will first see the login window.
+Proxmox Backup Server supports various languages and authentication back ends
+(*Realms*), both of which can be selected here.
+
+.. note:: For convenience, you can save the username on the client side, by
+  selecting the "Save User name" checkbox at the bottom of the window.
+
+
+GUI Overview
+------------
+
+.. image:: images/screenshots/pbs-gui-dashboard.png
+  :width: 250
+  :align: right
+  :alt: PBS GUI Dashboard
+
+The Proxmox Backup Server web interface consists of 3 main sections:
+
+* **Header**: At the top. This shows version information, and contains buttons to view
+  documentation, monitor running tasks, and logout.
+* **Sidebar**: On the left. This contains the configuration options for
+  the server.
+* **Configuration Panel**: In the center. This contains the control interface for the
+  configuration options in the *Sidebar*.
+
+
+Sidebar
+-------
+
+In the sidebar, on the left side of the page, you can see various items relating
+to specific management activities.
+
+
+Dashboard
+^^^^^^^^^
+
+The Dashboard shows a summary of activity and resource usage on the server.
+Specifically, this displays hardware usage, a summary of
+previous and currently running tasks, and subscription information.
+
+
+Configuration
+^^^^^^^^^^^^^
+
+The Configuration section contains some system configuration options, such as
+time and network configuration. It also contains the following subsections:
+
+* **User Management**: Add users and manage accounts
+* **Permissions**: Manage permissions for various users
+* **Remotes**: Add, edit and remove remotes (see :term:`Remote`)
+* **Sync Jobs**: Manage and run sync jobs to remotes
+* **Subscription**: Upload a subscription key and view subscription status
+
+
+Administration
+^^^^^^^^^^^^^^
+
+.. image:: images/screenshots/pbs-gui-administration-serverstatus.png
+  :width: 250
+  :align: right
+  :alt: Administration: Server Status overview
+
+The Administration section contains a top panel, with further administration
+tasks and information. These are:
+
+* **ServerStatus**: Provides access to the console, power options, and various
+  resource usage statistics
+* **Services**: Manage and monitor system services
+* **Updates**: An interface for upgrading packages
+* **Syslog**: View log messages from the server
+* **Tasks**: Task history with multiple filter options
+
+.. image:: images/screenshots/pbs-gui-disks.png
+  :width: 250
+  :align: right
+  :alt: Administration: Disks
+
+The administration menu item also contains a disk management subsection:
+
+* **Disks**: View information on available disks
+
+  * **Directory**: Create and view information on *ext4* and *xfs* disks
+  * **ZFS**: Create and view information on *ZFS* disks 
+
+
+Datastore
+^^^^^^^^^
+
+.. image:: images/screenshots/pbs-gui-datastore.png
+  :width: 250
+  :align: right
+  :alt: Datastore Configuration
+
+The Datastore section provides an interface for creating and managing
+datastores. It contains a subsection for each datastore on the system, in
+which you can use the top panel to view:
+
+* **Content**: Information on the datastore's backup groups and their respective
+  contents
+* **Statistics**: Usage statistics for the datastore
+* **Permissions**: View and manage permissions for the datastore

docs/index.rst

@@ -22,7 +22,16 @@ in the section entitled "GNU Free Documentation License".
 
    introduction.rst
    installation.rst
-   administration-guide.rst
+   terminology.rst
+   gui.rst
+   storage.rst
+   network-management.rst
+   user-management.rst
+   managing-remotes.rst
+   maintenance.rst
+   backup-client.rst
+   pve-integration.rst
+   pxar-tool.rst
    sysadmin.rst
    faq.rst
 

docs/installation.rst

@@ -5,6 +5,8 @@ Installation
 can either be installed with a graphical installer or on top of
 Debian_ from the provided package repository.
 
+.. include:: system-requirements.rst
+
 .. include:: package-repositories.rst
 
 Server installation
@@ -82,7 +84,7 @@ support, and a set of common and useful packages.
   when LVM_ or ZFS_ is used. The network configuration is completely up to you
   as well.
 
-.. note:: You can access the webinterface of the Proxmox Backup Server with
+.. note:: You can access the web interface of the Proxmox Backup Server with
   your web browser, using HTTPS on port 8007. For example at
  ``https://<ip-or-dns-name>:8007``
 
@@ -103,7 +105,7 @@ After configuring the
    still access the backups.
 
 .. note::
-  You can access the webinterface of the Proxmox Backup Server with your web
+  You can access the web interface of the Proxmox Backup Server with your web
   browser, using HTTPS on port 8007. For example at ``https://<ip-or-dns-name>:8007``
 
 Client installation

docs/introduction.rst

@@ -1,8 +1,8 @@
 Introduction
 ============
 
-What is Proxmox Backup Server
------------------------------
+What is Proxmox Backup Server?
+------------------------------
 
 Proxmox Backup Server is an enterprise-class, client-server backup software
 package that backs up :term:`virtual machine`\ s, :term:`container`\ s, and
@@ -10,12 +10,14 @@ physical hosts. It is specially optimized for the `Proxmox Virtual Environment`_
 platform and allows you to back up your data securely, even between remote
 sites, providing easy management with a web-based user interface.
 
-Proxmox Backup Server supports deduplication, compression, and authenticated
+It supports deduplication, compression, and authenticated
 encryption (AE_). Using :term:`Rust` as the implementation language guarantees high
 performance, low resource usage, and a safe, high-quality codebase.
 
-It features strong client-side encryption. Thus, it's possible to
-backup data to targets that are not fully trusted.
+Proxmox Backup uses state of the art cryptography for client communication and
+backup content :ref:`encryption <encryption>`. Encryption is done on the
+client side, making it safer to back up data to targets that are not fully
+trusted.
 
 
 Architecture
@@ -104,7 +106,7 @@ Software Stack
 
 Proxmox Backup Server consists of multiple components:
 
-* A server-daemon providing, among other things, a RESTfull API, super-fast
+* A server-daemon providing, among other things, a RESTful API, super-fast
   asynchronous tasks, lightweight usage statistic collection, scheduling
   events, strict separation of privileged and unprivileged execution
   environments
@@ -127,6 +129,7 @@ language.
 
 .. todo:: further explain the software stack
 
+
 Getting Help
 ------------
 
@@ -178,5 +181,28 @@ along with this program.  If not, see AGPL3_.
 History
 -------
 
-.. todo:: Add development History of the product
+Backup is, and always has been, a central aspect of IT administration.
+The need to recover from data loss is fundamental and only increases with
+virtualization.
 
+For this reason, we've been shipping a backup tool with Proxmox VE, from the
+beginning. This tool is called ``vzdump`` and is able to make
+consistent snapshots of running LXC containers and KVM virtual
+machines.
+
+However, ``vzdump`` only allows for full backups. While this is fine
+for small backups, it becomes a burden for users with large VMs. Both
+backup duration and storage usage are too high for this case, especially
+for users who want to keep many backups of the same VMs. To solve these
+problems, we needed to offer deduplication and incremental backups.
+
+Back in October 2018, development started. We investigated
+several technologies and frameworks and finally decided to use
+:term:`Rust` as the implementation language, in order to provide high speed and
+memory efficiency. The 2018-edition of Rust seemed promising for our
+requirements.
+
+In July 2020, we released the first beta version of Proxmox Backup
+Server, followed by the first stable version in November 2020. With support for
+incremental, fully deduplicated backups, Proxmox Backup significantly reduces
+network load and saves valuable storage space.

docs/local-zfs.rst

@@ -220,7 +220,7 @@ and you can install it using `apt-get`:
   # apt-get install zfs-zed
 
 To activate the daemon it is necessary to edit `/etc/zfs/zed.d/zed.rc` with your
-favourite editor, and uncomment the `ZED_EMAIL_ADDR` setting:
+favorite editor, and uncomment the `ZED_EMAIL_ADDR` setting:
 
 .. code-block:: console
 
@@ -262,7 +262,7 @@ to an external Storage.
 
 We strongly recommend to use enough memory, so that you normally do not
 run into low memory situations. Should you need or want to add swap, it is
-preferred to create a partition on a physical disk and use it as swapdevice.
+preferred to create a partition on a physical disk and use it as swap device.
 You can leave some space free for this purpose in the advanced options of the
 installer. Additionally, you can lower the `swappiness` value. 
 A good value for servers is 10:
@@ -312,6 +312,8 @@ You can disable compression at any time with:
 
 Only new blocks will be affected by this change.
 
+.. _local_zfs_special_device:
+
 ZFS Special Device
 ^^^^^^^^^^^^^^^^^^
 

docs/maintenance.rst

@@ -0,0 +1,13 @@
+Maintenance Tasks
+=================
+
+Garbage Collection
+------------------
+
+You can monitor and run :ref:`garbage collection <garbage-collection>` on the
+Proxmox Backup Server using the ``garbage-collection`` subcommand of
+``proxmox-backup-manager``. You can use the ``start`` subcommand to manually start garbage
+collection on an entire datastore and the ``status`` subcommand to see
+attributes relating to the :ref:`garbage collection <garbage-collection>`.
+
+.. todo:: Add section on verification

docs/managing-remotes.rst

@@ -0,0 +1,82 @@
+Managing Remotes
+================
+
+.. _backup_remote:
+
+:term:`Remote`
+--------------
+
+A remote refers to a separate Proxmox Backup Server installation and a user on that
+installation, from which you can `sync` datastores to a local datastore with a
+`Sync Job`. You can configure remotes in the web interface, under **Configuration
+-> Remotes**. Alternatively, you can use the ``remote`` subcommand. The
+configuration information for remotes is stored in the file
+``/etc/proxmox-backup/remote.cfg``.
+
+.. image:: images/screenshots/pbs-gui-remote-add.png
+  :align: right
+  :alt: Add a remote
+
+To add a remote, you need its hostname or IP, a userid and password on the
+remote, and its certificate fingerprint. To get the fingerprint, use the
+``proxmox-backup-manager cert info`` command on the remote, or navigate to
+**Dashboard** in the remote's web interface and select **Show Fingerprint**.
+
+.. code-block:: console
+
+  # proxmox-backup-manager cert info |grep Fingerprint
+  Fingerprint (sha256): 64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe
+
+Using the information specified above, you can add a remote from the **Remotes**
+configuration panel, or by using the command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager remote create pbs2 --host pbs2.mydomain.example --userid sync@pam --password 'SECRET' --fingerprint 64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe
+
+Use the ``list``, ``show``, ``update``, ``remove`` subcommands of
+``proxmox-backup-manager remote`` to manage your remotes:
+
+.. code-block:: console
+
+  # proxmox-backup-manager remote update pbs2 --host pbs2.example
+  # proxmox-backup-manager remote list
+  ┌──────┬──────────────┬──────────┬───────────────────────────────────────────┬─────────┐
+  │ name │ host         │ userid   │ fingerprint                               │ comment │
+  ╞══════╪══════════════╪══════════╪═══════════════════════════════════════════╪═════════╡
+  │ pbs2 │ pbs2.example │ sync@pam │64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe │         │
+  └──────┴──────────────┴──────────┴───────────────────────────────────────────┴─────────┘
+  # proxmox-backup-manager remote remove pbs2
+
+
+.. _syncjobs:
+
+Sync Jobs
+---------
+
+.. image:: images/screenshots/pbs-gui-syncjob-add.png
+  :align: right
+  :alt: Add a Sync Job
+
+Sync jobs are configured to pull the contents of a datastore on a **Remote** to
+a local datastore. You can manage sync jobs under **Configuration -> Sync Jobs**
+in the web interface, or using the ``proxmox-backup-manager sync-job`` command.
+The configuration information for sync jobs is stored at
+``/etc/proxmox-backup/sync.cfg``. To create a new sync job, click the add button
+in the GUI, or use the ``create`` subcommand. After creating a sync job, you can
+either start it manually on the GUI or provide it with a schedule (see
+:ref:`calendar-events`) to run regularly.
+
+.. code-block:: console
+
+  # proxmox-backup-manager sync-job create pbs2-local --remote pbs2 --remote-store local --store local --schedule 'Wed 02:30'
+  # proxmox-backup-manager sync-job update pbs2-local --comment 'offsite'
+  # proxmox-backup-manager sync-job list
+  ┌────────────┬───────┬────────┬──────────────┬───────────┬─────────┐
+  │ id         │ store │ remote │ remote-store │ schedule  │ comment │
+  ╞════════════╪═══════╪════════╪══════════════╪═══════════╪═════════╡
+  │ pbs2-local │ local │ pbs2   │ local        │ Wed 02:30 │ offsite │
+  └────────────┴───────┴────────┴──────────────┴───────────┴─────────┘
+  # proxmox-backup-manager sync-job remove pbs2-local
+
+

docs/network-management.rst

@@ -0,0 +1,88 @@
+Network Management
+==================
+
+Proxmox Backup Server provides both a web interface and a command line tool for
+network configuration. You can find the configuration options in the web
+interface under the **Network Interfaces** section of the **Configuration** menu
+tree item. The command line tool is accessed via the ``network`` subcommand.
+These interfaces allow you to carry out some basic network management tasks,
+such as adding, configuring, and removing network interfaces.
+
+.. note:: Any changes made to the network configuration are not
+  applied, until you click on **Apply Configuration** or enter the ``network
+  reload`` command. This allows you to make many changes at once. It also allows
+  you to ensure that your changes are correct before applying them, as making a
+  mistake here can render the server inaccessible over the network.
+
+To get a list of available interfaces, use the following command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network list
+  ┌───────┬────────┬───────────┬────────┬─────────────┬──────────────┬──────────────┐
+  │ name  │ type   │ autostart │ method │ address     │ gateway      │ ports/slaves │
+  ╞═══════╪════════╪═══════════╪════════╪═════════════╪══════════════╪══════════════╡
+  │ bond0 │ bond   │         1 │ static │ x.x.x.x/x   │ x.x.x.x      │ ens18 ens19  │
+  ├───────┼────────┼───────────┼────────┼─────────────┼──────────────┼──────────────┤
+  │ ens18 │ eth    │         1 │ manual │             │              │              │
+  ├───────┼────────┼───────────┼────────┼─────────────┼──────────────┼──────────────┤
+  │ ens19 │ eth    │         1 │ manual │             │              │              │
+  └───────┴────────┴───────────┴────────┴─────────────┴──────────────┴──────────────┘
+
+.. image:: images/screenshots/pbs-gui-network-create-bond.png
+  :align: right
+  :alt: Add a network interface
+
+To add a new network interface, use the ``create`` subcommand with the relevant
+parameters. For example, you may want to set up a bond, for the purpose of
+network redundancy. The following command shows a template for creating the bond shown
+in the list above:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network create bond0 --type bond --bond_mode active-backup --slaves ens18,ens19 --autostart true --cidr x.x.x.x/x --gateway x.x.x.x
+
+You can make changes to the configuration of a network interface with the
+``update`` subcommand:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network update bond0 --cidr y.y.y.y/y
+
+You can also remove a network interface:
+
+.. code-block:: console
+
+   # proxmox-backup-manager network remove bond0
+
+The pending changes for the network configuration file will appear at the bottom of the
+web interface. You can also view these changes, by using the command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network changes
+
+If you would like to cancel all changes at this point, you can either click on
+the **Revert** button or use the following command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network revert
+
+If you are happy with the changes and would like to write them into the
+configuration file, select **Apply Configuration**. The corresponding command
+is:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network reload
+
+.. note:: This command and corresponding GUI button rely on the ``ifreload``
+  command, from the package ``ifupdown2``. This package is included within the
+  Proxmox Backup Server installation, however, you may have to install it yourself,
+  if you have installed Proxmox Backup Server on top of Debian or Proxmox VE.
+
+You can also configure DNS settings, from the **DNS** section
+of **Configuration** or by using the ``dns`` subcommand of
+``proxmox-backup-manager``.
+

docs/pve-integration.rst

@@ -0,0 +1,49 @@
+.. _pve-integration:
+
+`Proxmox VE`_ Integration
+-------------------------
+
+You need to define a new storage with type 'pbs' on your `Proxmox VE`_
+node. The following example uses ``store2`` as storage name, and
+assumes the server address is ``localhost``, and you want to connect
+as ``user1@pbs``.
+
+.. code-block:: console
+
+  # pvesm add pbs store2 --server localhost --datastore store2
+  # pvesm set store2 --username user1@pbs --password <secret>
+
+.. note:: If you would rather not pass your password as plain text, you can pass
+  the ``--password`` parameter, without any arguments. This will cause the
+  program to prompt you for a password upon entering the command.
+
+If your backup server uses a self signed certificate, you need to add
+the certificate fingerprint to the configuration. You can get the
+fingerprint by running the following command on the backup server:
+
+.. code-block:: console
+
+  # proxmox-backup-manager cert info | grep Fingerprint
+  Fingerprint (sha256): 64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe
+
+Please add that fingerprint to your configuration to establish a trust
+relationship:
+
+.. code-block:: console
+
+  # pvesm set store2 --fingerprint  64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe
+
+After that you should be able to see storage status with:
+
+.. code-block:: console
+
+  # pvesm status --storage store2
+  Name             Type     Status           Total            Used       Available        %
+  store2            pbs     active      3905109820      1336687816      2568422004   34.23%
+
+Having added the PBS datastore to `Proxmox VE`_, you can backup VMs and
+containers in the same way you would for any other storage device within the
+environment (see `PVE Admin Guide: Backup and Restore
+<https://pve.proxmox.com/pve-docs/pve-admin-guide.html#chapter_vzdump>`_.
+
+

docs/pxar-tool.rst

@@ -0,0 +1,5 @@
+pxar Command Line Tool
+======================
+
+.. include:: pxar/description.rst
+

docs/storage.rst

@@ -0,0 +1,244 @@
+Storage
+=======
+
+Disk Management
+---------------
+
+.. image:: images/screenshots/pbs-gui-disks.png
+  :align: right
+  :alt: List of disks
+
+Proxmox Backup Server comes with a set of disk utilities, which are
+accessed using the ``disk`` subcommand. This subcommand allows you to initialize
+disks, create various filesystems, and get information about the disks.
+
+To view the disks connected to the system, navigate to **Administration ->
+Disks** in the web interface or use the ``list`` subcommand of
+``disk``:
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk list
+  ┌──────┬────────┬─────┬───────────┬─────────────┬───────────────┬─────────┬────────┐
+  │ name │ used   │ gpt │ disk-type │        size │ model         │ wearout │ status │
+  ╞══════╪════════╪═════╪═══════════╪═════════════╪═══════════════╪═════════╪════════╡
+  │ sda  │ lvm    │   1 │ hdd       │ 34359738368 │ QEMU_HARDDISK │       - │ passed │
+  ├──────┼────────┼─────┼───────────┼─────────────┼───────────────┼─────────┼────────┤
+  │ sdb  │ unused │   1 │ hdd       │ 68719476736 │ QEMU_HARDDISK │       - │ passed │
+  ├──────┼────────┼─────┼───────────┼─────────────┼───────────────┼─────────┼────────┤
+  │ sdc  │ unused │   1 │ hdd       │ 68719476736 │ QEMU_HARDDISK │       - │ passed │
+  └──────┴────────┴─────┴───────────┴─────────────┴───────────────┴─────────┴────────┘
+
+To initialize a disk with a new GPT, use the ``initialize`` subcommand:
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk initialize sdX
+
+.. image:: images/screenshots/pbs-gui-disks-dir-create.png
+  :align: right
+  :alt: Create a directory
+
+You can create an ``ext4`` or ``xfs`` filesystem on a disk using ``fs
+create``, or by navigating to **Administration -> Disks -> Directory** in the
+web interface and creating one from there. The following command creates an
+``ext4`` filesystem and passes the ``--add-datastore`` parameter, in order to
+automatically create a datastore on the disk (in this case ``sdd``). This will
+create a datastore at the location ``/mnt/datastore/store1``:
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk fs create store1 --disk sdd --filesystem ext4 --add-datastore true
+
+.. image:: images/screenshots/pbs-gui-disks-zfs-create.png
+  :align: right
+  :alt: Create ZFS
+
+You can also create a ``zpool`` with various raid levels from **Administration
+-> Disks -> Zpool** in the web interface, or by using ``zpool create``. The command
+below creates a mirrored ``zpool`` using two disks (``sdb`` & ``sdc``) and
+mounts it on the root directory (default):
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk zpool create zpool1 --devices sdb,sdc --raidlevel mirror
+
+.. note:: You can also pass the ``--add-datastore`` parameter here, to automatically
+  create a datastore from the disk.
+
+You can use ``disk fs list`` and ``disk zpool list`` to keep track of your
+filesystems and zpools respectively.
+
+Proxmox Backup Server uses the package smartmontools. This is a set of tools
+used to monitor and control the S.M.A.R.T. system for local hard disks. If a
+disk supports S.M.A.R.T. capability, and you have this enabled, you can
+display S.M.A.R.T. attributes from the web interface or by using the command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk smart-attributes sdX
+
+.. note:: This functionality may also be accessed directly through the use of
+  the ``smartctl`` command, which comes as part of the smartmontools package
+  (see ``man smartctl`` for more details).
+
+
+.. _datastore_intro:
+
+:term:`DataStore`
+-----------------
+
+A datastore refers to a location at which backups are stored. The current
+implementation uses a directory inside a standard Unix file system (``ext4``,
+``xfs`` or ``zfs``) to store the backup data.
+
+Datastores are identified by a simple *ID*. You can configure this
+when setting up the datastore. The configuration information for datastores
+is stored in the file ``/etc/proxmox-backup/datastore.cfg``.
+
+.. note:: The `File Layout`_ requires the file system to support at least *65538*
+   subdirectories per directory. That number comes from the 2\ :sup:`16`
+   pre-created chunk namespace directories, and the ``.`` and ``..`` default
+   directory entries. This requirement excludes certain filesystems and
+   filesystem configuration from being supported for a datastore. For example,
+   ``ext3`` as a whole or ``ext4`` with the ``dir_nlink`` feature manually disabled.
+
+
+Datastore Configuration
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. image:: images/screenshots/pbs-gui-datastore.png
+  :align: right
+  :alt: Datastore Overview
+
+You can configure multiple datastores. Minimum one datastore needs to be
+configured. The datastore is identified by a simple *name* and points to a
+directory on the filesystem. Each datastore also has associated retention
+settings of how many backup snapshots for each interval of ``hourly``,
+``daily``, ``weekly``, ``monthly``, ``yearly`` as well as a time-independent
+number of backups to keep in that store. :ref:`backup-pruning` and
+:ref:`garbage collection <garbage-collection>` can also be configured to run
+periodically based on a configured schedule (see :ref:`calendar-events`) per datastore.
+
+
+Creating a Datastore
+^^^^^^^^^^^^^^^^^^^^
+.. image:: images/screenshots/pbs-gui-datastore-create-general.png
+  :align: right
+  :alt: Create a datastore
+
+You can create a new datastore from the web GUI, by navigating to **Datastore** in
+the menu tree and clicking **Create**. Here:
+
+* *Name* refers to the name of the datastore
+* *Backing Path* is the path to the directory upon which you want to create the
+  datastore
+* *GC Schedule* refers to the time and intervals at which garbage collection
+  runs
+* *Prune Schedule* refers to the frequency at which pruning takes place
+* *Prune Options* set the amount of backups which you would like to keep (see :ref:`backup-pruning`).
+
+Alternatively you can create a new datastore from the command line. The
+following command creates a new datastore called ``store1`` on :file:`/backup/disk1/store1`
+
+.. code-block:: console
+
+  # proxmox-backup-manager datastore create store1 /backup/disk1/store1
+
+
+Managing Datastores
+^^^^^^^^^^^^^^^^^^^
+
+To list existing datastores from the command line run:
+
+.. code-block:: console
+
+  # proxmox-backup-manager datastore list
+  ┌────────┬──────────────────────┬─────────────────────────────┐
+  │ name   │ path                 │ comment                     │
+  ╞════════╪══════════════════════╪═════════════════════════════╡
+  │ store1 │ /backup/disk1/store1 │ This is my default storage. │
+  └────────┴──────────────────────┴─────────────────────────────┘
+
+You can change the garbage collection and prune settings of a datastore, by
+editing the datastore from the GUI or by using the ``update`` subcommand. For
+example, the below command changes the garbage collection schedule using the
+``update`` subcommand and prints the properties of the datastore with the
+``show`` subcommand:
+
+.. code-block:: console
+
+  # proxmox-backup-manager datastore update store1 --gc-schedule 'Tue 04:27'
+  # proxmox-backup-manager datastore show store1
+  ┌────────────────┬─────────────────────────────┐
+  │ Name           │ Value                       │
+  ╞════════════════╪═════════════════════════════╡
+  │ name           │ store1                      │
+  ├────────────────┼─────────────────────────────┤
+  │ path           │ /backup/disk1/store1        │
+  ├────────────────┼─────────────────────────────┤
+  │ comment        │ This is my default storage. │
+  ├────────────────┼─────────────────────────────┤
+  │ gc-schedule    │ Tue 04:27                   │
+  ├────────────────┼─────────────────────────────┤
+  │ keep-last      │ 7                           │
+  ├────────────────┼─────────────────────────────┤
+  │ prune-schedule │ daily                       │
+  └────────────────┴─────────────────────────────┘
+
+Finally, it is possible to remove the datastore configuration:
+
+.. code-block:: console
+
+  # proxmox-backup-manager datastore remove store1
+
+.. note:: The above command removes only the datastore configuration. It does
+   not delete any data from the underlying directory.
+
+
+File Layout
+^^^^^^^^^^^
+
+After creating a datastore, the following default layout will appear:
+
+.. code-block:: console
+
+  # ls -arilh /backup/disk1/store1
+  276493 -rw-r--r-- 1 backup backup       0 Jul  8 12:35 .lock
+  276490 drwxr-x--- 1 backup backup 1064960 Jul  8 12:35 .chunks
+
+`.lock` is an empty file used for process locking.
+
+The `.chunks` directory contains folders, starting from `0000` and taking hexadecimal values until `ffff`. These
+directories will store the chunked data after a backup operation has been executed.
+
+.. code-block:: console
+
+ # ls -arilh /backup/disk1/store1/.chunks
+ 545824 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 ffff
+ 545823 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fffe
+ 415621 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fffd
+ 415620 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fffc
+ 353187 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fffb
+ 344995 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fffa
+ 144079 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fff9
+ 144078 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fff8
+ 144077 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fff7
+ ...
+ 403180 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 000c
+ 403179 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 000b
+ 403177 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 000a
+ 402530 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0009
+ 402513 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0008
+ 402509 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0007
+ 276509 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0006
+ 276508 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0005
+ 276507 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0004
+ 276501 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0003
+ 276499 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0002
+ 276498 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0001
+ 276494 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0000
+ 276489 drwxr-xr-x 3 backup backup 4.0K Jul  8 12:35 ..
+ 276490 drwxr-x--- 1 backup backup 1.1M Jul  8 12:35 .
+
+

docs/system-requirements.rst

@@ -0,0 +1,57 @@
+System Requirements
+-------------------
+
+We recommend using high quality server hardware when running Proxmox Backup in
+production. To further decrease the impact of a failed host, you can set up
+periodic, efficient, incremental :ref:`datastore synchronization <syncjobs>`
+from other Proxmox Backup Server instances.
+
+Minimum Server Requirements, for Evaluation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+These minimum requirements are for evaluation purposes only and should not be
+used in production.
+
+* CPU: 64bit (*x86-64* or *AMD64*), 2+ Cores
+
+* Memory (RAM): 2 GB RAM
+
+* Hard drive: more than 8GB of space.
+
+* Network card (NIC)
+
+
+Recommended Server System Requirements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* CPU: Modern AMD or Intel 64-bit based CPU, with at least 4 cores
+
+* Memory: minimum 4 GiB for the OS, filesystem cache and Proxmox Backup Server
+  daemons. Add at least another GiB per TiB storage space.
+
+* OS storage:
+
+  * 32 GiB, or more, free storage space
+  * Use a hardware RAID with battery protected write cache (*BBU*) or a
+    redundant ZFS setup (ZFS is not compatible with a hardware RAID
+    controller).
+
+* Backup storage:
+
+  * Use only SSDs, for best results
+  * If HDDs are used: Using a metadata cache is highly recommended, for example,
+    add a ZFS :ref:`special device mirror <local_zfs_special_device>`.
+
+* Redundant Multi-GBit/s network interface cards (NICs)
+
+
+Supported Web Browsers for Accessing the Web Interface
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To access the server's web-based user interface, we recommend using one of the
+following browsers:
+
+* Firefox, a release from the current year, or the latest Extended Support Release
+* Chrome, a release from the current year
+* Microsoft's currently supported version of Edge
+* Safari, a release from the current year

docs/terminology.rst

@@ -0,0 +1,118 @@
+Terminology
+===========
+
+Backup Content
+--------------
+
+When doing deduplication, there are different strategies to get
+optimal results in terms of performance and/or deduplication rates.
+Depending on the type of data, it can be split into *fixed* or *variable*
+sized chunks.
+
+Fixed sized chunking requires minimal CPU power, and is used to
+backup virtual machine images.
+
+Variable sized chunking needs more CPU power, but is essential to get
+good deduplication rates for file archives.
+
+The Proxmox Backup Server supports both strategies.
+
+
+Image Archives: ``<name>.img``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is used for virtual machine images and other large binary
+data. Content is split into fixed-sized chunks.
+
+
+File Archives: ``<name>.pxar``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. see https://moinakg.wordpress.com/2013/06/22/high-performance-content-defined-chunking/
+
+A file archive stores a full directory tree. Content is stored using
+the :ref:`pxar-format`, split into variable-sized chunks. The format
+is optimized to achieve good deduplication rates.
+
+
+Binary Data (BLOBs)
+~~~~~~~~~~~~~~~~~~~
+
+This type is used to store smaller (< 16MB) binary data such as
+configuration files. Larger files should be stored as image archive.
+
+.. caution:: Please do not store all files as BLOBs. Instead, use the
+   file archive to store whole directory trees.
+
+
+Catalog File: ``catalog.pcat1``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The catalog file is an index for file archives. It contains
+the list of files and is used to speed up search operations.
+
+
+The Manifest: ``index.json``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The manifest contains the list of all backup files, their
+sizes and checksums. It is used to verify the consistency of a
+backup.
+
+
+Backup Type
+-----------
+
+The backup server groups backups by *type*, where *type* is one of:
+
+``vm``
+    This type is used for :term:`virtual machine`\ s. Typically
+    consists of the virtual machine's configuration file and an image archive
+    for each disk.
+
+``ct``
+    This type is used for :term:`container`\ s. Consists of the container's
+    configuration and a single file archive for the filesystem content.
+
+``host``
+    This type is used for backups created from within the backed up machine.
+    Typically this would be a physical host but could also be a virtual machine
+    or container. Such backups may contain file and image archives, there are no restrictions in this regard.
+
+
+Backup ID
+---------
+
+A unique ID. Usually the virtual machine or container ID. ``host``
+type backups normally use the hostname.
+
+
+Backup Time
+-----------
+
+The time when the backup was made.
+
+
+Backup Group
+------------
+
+The tuple ``<type>/<ID>`` is called a backup group. Such a group
+may contain one or more backup snapshots.
+
+
+Backup Snapshot
+---------------
+
+The triplet ``<type>/<ID>/<time>`` is called a backup snapshot. It
+uniquely identifies a specific backup within a datastore.
+
+.. code-block:: console
+   :caption: Backup Snapshot Examples
+
+    vm/104/2019-10-09T08:01:06Z
+    host/elsa/2019-11-08T09:48:14Z
+
+As you can see, the time format is RFC3399_ with Coordinated
+Universal Time (UTC_, identified by the trailing *Z*).
+
+

docs/user-management.rst

@@ -0,0 +1,186 @@
+.. _user_mgmt:
+
+User Management
+===============
+
+
+User Configuration
+------------------
+
+.. image:: images/screenshots/pbs-gui-user-management.png
+  :align: right
+  :alt: User management
+
+Proxmox Backup Server supports several authentication realms, and you need to
+choose the realm when you add a new user. Possible realms are:
+
+:pam: Linux PAM standard authentication. Use this if you want to
+      authenticate as Linux system user (Users need to exist on the
+      system).
+
+:pbs: Proxmox Backup Server realm. This type stores hashed passwords in
+      ``/etc/proxmox-backup/shadow.json``.
+
+After installation, there is a single user ``root@pam``, which
+corresponds to the Unix superuser. User configuration information is stored in the file
+``/etc/proxmox-backup/user.cfg``. You can use the
+``proxmox-backup-manager`` command line tool to list or manipulate
+users:
+
+.. code-block:: console
+
+  # proxmox-backup-manager user list
+  ┌─────────────┬────────┬────────┬───────────┬──────────┬────────────────┬────────────────────┐
+  │ userid      │ enable │ expire │ firstname │ lastname │ email          │ comment            │
+  ╞═════════════╪════════╪════════╪═══════════╪══════════╪════════════════╪════════════════════╡
+  │ root@pam    │      1 │        │           │          │                │ Superuser          │
+  └─────────────┴────────┴────────┴───────────┴──────────┴────────────────┴────────────────────┘
+
+.. image:: images/screenshots/pbs-gui-user-management-add-user.png
+  :align: right
+  :alt: Add a new user
+
+The superuser has full administration rights on everything, so you
+normally want to add other users with less privileges. You can create a new
+user with the ``user create`` subcommand or through the web interface, under
+**Configuration -> User Management**. The ``create`` subcommand lets you specify
+many options like ``--email`` or ``--password``. You can update or change any
+user properties using the ``update`` subcommand later (**Edit** in the GUI):
+
+
+.. code-block:: console
+
+  # proxmox-backup-manager user create john@pbs --email john@example.com
+  # proxmox-backup-manager user update john@pbs --firstname John --lastname Smith
+  # proxmox-backup-manager user update john@pbs --comment "An example user."
+
+.. todo:: Mention how to set password without passing plaintext password as cli argument.
+
+
+The resulting user list looks like this:
+
+.. code-block:: console
+
+  # proxmox-backup-manager user list
+  ┌──────────┬────────┬────────┬───────────┬──────────┬──────────────────┬──────────────────┐
+  │ userid   │ enable │ expire │ firstname │ lastname │ email            │ comment          │
+  ╞══════════╪════════╪════════╪═══════════╪══════════╪══════════════════╪══════════════════╡
+  │ john@pbs │      1 │        │ John      │ Smith    │ john@example.com │ An example user. │
+  ├──────────┼────────┼────────┼───────────┼──────────┼──────────────────┼──────────────────┤
+  │ root@pam │      1 │        │           │          │                  │ Superuser        │
+  └──────────┴────────┴────────┴───────────┴──────────┴──────────────────┴──────────────────┘
+
+Newly created users do not have any permissions. Please read the next
+section to learn how to set access permissions.
+
+If you want to disable a user account, you can do that by setting ``--enable`` to ``0``
+
+.. code-block:: console
+
+  # proxmox-backup-manager user update john@pbs --enable 0
+
+Or completely remove the user with:
+
+.. code-block:: console
+
+  # proxmox-backup-manager user remove john@pbs
+
+
+.. _user_acl:
+
+Access Control
+--------------
+
+By default new users do not have any permission. Instead you need to
+specify what is allowed and what is not. You can do this by assigning
+roles to users on specific objects like datastores or remotes. The
+following roles exist:
+
+**NoAccess**
+  Disable Access - nothing is allowed.
+
+**Admin**
+  Can do anything.
+
+**Audit**
+  Can view things, but is not allowed to change settings.
+
+**DatastoreAdmin**
+  Can do anything on datastores.
+
+**DatastoreAudit**
+  Can view datastore settings and list content. But
+  is not allowed to read the actual data.
+
+**DatastoreReader**
+  Can Inspect datastore content and can do restores.
+
+**DatastoreBackup**
+  Can backup and restore owned backups.
+
+**DatastorePowerUser**
+  Can backup, restore, and prune owned backups.
+
+**RemoteAdmin**
+  Can do anything on remotes.
+
+**RemoteAudit**
+  Can view remote settings.
+
+**RemoteSyncOperator**
+  Is allowed to read data from a remote.
+
+.. image:: images/screenshots/pbs-gui-permissions-add.png
+  :align: right
+  :alt: Add permissions for user
+
+Access permission information is stored in ``/etc/proxmox-backup/acl.cfg``. The
+file contains 5 fields, separated using a colon (':') as a delimiter. A typical
+entry takes the form:
+
+``acl:1:/datastore:john@pbs:DatastoreBackup``
+
+The data represented in each field is as follows:
+
+#. ``acl`` identifier
+#. A ``1`` or ``0``, representing whether propagation is enabled or disabled,
+   respectively
+#. The object on which the permission is set. This can be a specific object
+   (single datastore, remote, etc.) or a top level object, which with
+   propagation enabled, represents all children of the object also.
+#. The user for which the permission is set
+#. The role being set
+
+You can manage datastore permissions from **Configuration -> Permissions** in the
+web interface. Likewise, you can use the ``acl`` subcommand to manage and
+monitor user permissions from the command line. For example, the command below
+will add the user ``john@pbs`` as a **DatastoreAdmin** for the datastore
+``store1``, located at ``/backup/disk1/store1``:
+
+.. code-block:: console
+
+  # proxmox-backup-manager acl update /datastore/store1 DatastoreAdmin --userid john@pbs
+
+You can monitor the roles of each user using the following command:
+
+.. code-block:: console
+
+   # proxmox-backup-manager acl list
+   ┌──────────┬──────────────────┬───────────┬────────────────┐
+   │ ugid     │ path             │ propagate │ roleid         │
+   ╞══════════╪══════════════════╪═══════════╪════════════════╡
+   │ john@pbs │ /datastore/disk1 │         1 │ DatastoreAdmin │
+   └──────────┴──────────────────┴───────────┴────────────────┘
+
+A single user can be assigned multiple permission sets for different datastores.
+
+.. Note::
+  Naming convention is important here. For datastores on the host,
+  you must use the convention ``/datastore/{storename}``. For example, to set
+  permissions for a datastore mounted at ``/mnt/backup/disk4/store2``, you would use
+  ``/datastore/store2`` for the path. For remote stores, use the convention
+  ``/remote/{remote}/{storename}``, where ``{remote}`` signifies the name of the
+  remote (see `Remote` below) and ``{storename}`` is the name of the datastore on
+  the remote.
+
+

src/api2.rs

@@ -7,6 +7,7 @@ pub mod reader;
 pub mod status;
 pub mod types;
 pub mod version;
+pub mod ping;
 pub mod pull;
 mod helpers;
 
@@ -22,6 +23,7 @@ pub const SUBDIRS: SubdirMap = &[
     ("backup", &backup::ROUTER),
     ("config", &config::ROUTER),
     ("nodes", &NODES_ROUTER),
+    ("ping", &ping::ROUTER),
     ("pull", &pull::ROUTER),
     ("reader", &reader::ROUTER),
     ("status", &status::ROUTER),

src/api2/access.rs

@@ -10,6 +10,7 @@ use proxmox::{http_err, list_subdirs_api_method};
 use crate::tools::ticket::{self, Empty, Ticket};
 use crate::auth_helpers::*;
 use crate::api2::types::*;
+use crate::tools::{FileLogOptions, FileLogger};
 
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::acl::{PRIVILEGES, PRIV_PERMISSIONS_MODIFY};
@@ -138,14 +139,22 @@ fn create_ticket(
     path: Option<String>,
     privs: Option<String>,
     port: Option<u16>,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
+    let logger_options = FileLogOptions {
+        append: true,
+        prefix_time: true,
+        ..Default::default()
+    };
+    let mut auth_log = FileLogger::new("/var/log/proxmox-backup/api/auth.log", logger_options)?;
+
     match authenticate_user(&username, &password, path, privs, port) {
         Ok(true) => {
             let ticket = Ticket::new("PBS", &username)?.sign(private_auth_key(), None)?;
 
             let token = assemble_csrf_prevention_token(csrf_secret(), &username);
 
-            log::info!("successful auth for user '{}'", username);
+            auth_log.log(format!("successful auth for user '{}'", username));
 
             Ok(json!({
                 "username": username,
@@ -157,8 +166,20 @@ fn create_ticket(
             "username": username,
         })),
         Err(err) => {
-            let client_ip = "unknown"; // $rpcenv->get_client_ip() || '';
-            log::error!("authentication failure; rhost={} user={} msg={}", client_ip, username, err.to_string());
+            let client_ip = match rpcenv.get_client_ip().map(|addr| addr.ip()) {
+                Some(ip) => format!("{}", ip),
+                None => "unknown".into(),
+            };
+
+            let msg = format!(
+                "authentication failure; rhost={} user={} msg={}",
+                client_ip,
+                username,
+                err.to_string()
+            );
+            auth_log.log(&msg);
+            log::error!("{}", msg);
+
             Err(http_err!(UNAUTHORIZED, "permission check failed."))
         }
     }

src/api2/admin.rs

@@ -3,10 +3,12 @@ use proxmox::list_subdirs_api_method;
 
 pub mod datastore;
 pub mod sync;
+pub mod verify;
 
 const SUBDIRS: SubdirMap = &[
     ("datastore", &datastore::ROUTER),
-    ("sync", &sync::ROUTER)
+    ("sync", &sync::ROUTER),
+    ("verify", &verify::ROUTER)
 ];
 
 pub const ROUTER: Router = Router::new()

src/api2/admin/datastore.rs

@@ -2,6 +2,8 @@ use std::collections::{HashSet, HashMap};
 use std::ffi::OsStr;
 use std::os::unix::ffi::OsStrExt;
 use std::sync::{Arc, Mutex};
+use std::path::{Path, PathBuf};
+use std::pin::Pin;
 
 use anyhow::{bail, format_err, Error};
 use futures::*;
@@ -16,10 +18,9 @@ use proxmox::api::{
 use proxmox::api::router::SubdirMap;
 use proxmox::api::schema::*;
 use proxmox::tools::fs::{replace_file, CreateOptions};
-use proxmox::try_block;
 use proxmox::{http_err, identity, list_subdirs_api_method, sortable};
 
-use pxar::accessor::aio::Accessor;
+use pxar::accessor::aio::{Accessor, FileContents, FileEntry};
 use pxar::EntryKind;
 
 use crate::api2::types::*;
@@ -29,7 +30,12 @@ use crate::config::datastore;
 use crate::config::cached_user_info::CachedUserInfo;
 
 use crate::server::WorkerTask;
-use crate::tools::{self, AsyncReaderStream, WrappedReaderStream};
+use crate::tools::{
+    self,
+    zip::{ZipEncoder, ZipEntry},
+    AsyncChannelWriter, AsyncReaderStream, WrappedReaderStream,
+};
+
 use crate::config::acl::{
     PRIV_DATASTORE_AUDIT,
     PRIV_DATASTORE_MODIFY,
@@ -165,8 +171,8 @@ fn list_groups(
 
         let list_all = (user_privs & PRIV_DATASTORE_AUDIT) != 0;
         let owner = datastore.get_owner(group)?;
-        if !list_all {
-            if owner != userid { continue; }
+        if !list_all && owner != userid {
+            continue;
         }
 
         let result_item = GroupListItem {
@@ -356,8 +362,8 @@ pub fn list_snapshots (
         let list_all = (user_privs & PRIV_DATASTORE_AUDIT) != 0;
         let owner = datastore.get_owner(group)?;
 
-        if !list_all {
-            if owner != userid { continue; }
+        if !list_all && owner != userid {
+            continue;
         }
 
         let mut size = None;
@@ -417,6 +423,37 @@ pub fn list_snapshots (
     Ok(snapshots)
 }
 
+// returns a map from type to (group_count, snapshot_count)
+fn get_snaphots_count(store: &DataStore) -> Result<HashMap<String, (usize, usize)>, Error> {
+    let base_path = store.base_path();
+    let backup_list = BackupInfo::list_backups(&base_path)?;
+    let mut groups = HashSet::new();
+    let mut result: HashMap<String, (usize, usize)> = HashMap::new();
+    for info in backup_list {
+        let group = info.backup_dir.group();
+
+        let id = group.backup_id();
+        let backup_type = group.backup_type();
+
+        let mut new_id = false;
+
+        if groups.insert(format!("{}-{}", &backup_type, &id)) {
+            new_id = true;
+        }
+
+        if let Some(mut counts) = result.get_mut(backup_type) {
+            counts.1 += 1;
+            if new_id {
+                counts.0 +=1;
+            }
+        } else {
+            result.insert(backup_type.to_string(), (1, 1));
+        }
+    }
+
+    Ok(result)
+}
+
 #[api(
     input: {
         properties: {
@@ -426,7 +463,21 @@ pub fn list_snapshots (
         },
     },
     returns: {
-        type: StorageStatus,
+        description: "The overall Datastore status and information.",
+        type: Object,
+        properties: {
+            storage: {
+                type: StorageStatus,
+            },
+            counts: {
+                description: "Group and Snapshot counts per Type",
+                type: Object,
+                properties: { },
+            },
+            "gc-status": {
+                type: GarbageCollectionStatus,
+            },
+        },
     },
     access: {
         permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_BACKUP, true),
@@ -437,9 +488,19 @@ pub fn status(
     store: String,
     _info: &ApiMethod,
     _rpcenv: &mut dyn RpcEnvironment,
-) -> Result<StorageStatus, Error> {
+) -> Result<Value, Error> {
     let datastore = DataStore::lookup_datastore(&store)?;
-    crate::tools::disks::disk_usage(&datastore.base_path())
+    let storage_status = crate::tools::disks::disk_usage(&datastore.base_path())?;
+    let counts = get_snaphots_count(&datastore)?;
+    let gc_status = datastore.last_gc_status();
+
+    let res = json!({
+        "storage": storage_status,
+        "counts": counts,
+        "gc-status": gc_status,
+    });
+
+    Ok(res)
 }
 
 #[api(
@@ -486,17 +547,20 @@ pub fn verify(
 
     let mut backup_dir = None;
     let mut backup_group = None;
+    let mut worker_type = "verify";
 
     match (backup_type, backup_id, backup_time) {
         (Some(backup_type), Some(backup_id), Some(backup_time)) => {
-            worker_id = format!("{}_{}_{}_{:08X}", store, backup_type, backup_id, backup_time);
+            worker_id = format!("{}:{}/{}/{:08X}", store, backup_type, backup_id, backup_time);
             let dir = BackupDir::new(backup_type, backup_id, backup_time)?;
             backup_dir = Some(dir);
+            worker_type = "verify_snapshot";
         }
         (Some(backup_type), Some(backup_id), None) => {
-            worker_id = format!("{}_{}_{}", store, backup_type, backup_id);
+            worker_id = format!("{}:{}/{}", store, backup_type, backup_id);
             let group = BackupGroup::new(backup_type, backup_id);
             backup_group = Some(group);
+            worker_type = "verify_group";
         }
         (None, None, None) => {
             worker_id = store.clone();
@@ -508,17 +572,25 @@ pub fn verify(
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
 
     let upid_str = WorkerTask::new_thread(
-        "verify",
+        worker_type,
         Some(worker_id.clone()),
         userid,
         to_stdout,
         move |worker| {
             let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*16)));
             let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
+            let filter = |_backup_info: &BackupInfo| { true };
 
             let failed_dirs = if let Some(backup_dir) = backup_dir {
                 let mut res = Vec::new();
-                if !verify_backup_dir(datastore, &backup_dir, verified_chunks, corrupt_chunks, worker.clone())? {
+                if !verify_backup_dir(
+                    datastore,
+                    &backup_dir,
+                    verified_chunks,
+                    corrupt_chunks,
+                    worker.clone(),
+                    worker.upid().clone(),
+                )? {
                     res.push(backup_dir.to_string());
                 }
                 res
@@ -530,10 +602,12 @@ pub fn verify(
                     corrupt_chunks,
                     None,
                     worker.clone(),
+                    worker.upid(),
+                    &filter,
                 )?;
                 failed_dirs
             } else {
-                verify_all_backups(datastore, worker.clone())?
+                verify_all_backups(datastore, worker.clone(), worker.upid(), &filter)?
             };
             if failed_dirs.len() > 0 {
                 worker.log("Failed to verify following snapshots:");
@@ -651,7 +725,7 @@ fn prune(
         keep_yearly: param["keep-yearly"].as_u64(),
     };
 
-    let worker_id = format!("{}_{}_{}", store, backup_type, backup_id);
+    let worker_id = format!("{}:{}/{}", store, backup_type, backup_id);
 
     let mut prune_result = Vec::new();
 
@@ -684,53 +758,52 @@ fn prune(
     // We use a WorkerTask just to have a task log, but run synchrounously
     let worker = WorkerTask::new("prune", Some(worker_id), Userid::root_userid().clone(), true)?;
 
-    let result = try_block! {
-        if keep_all {
-            worker.log("No prune selection - keeping all files.");
-        } else {
-            worker.log(format!("retention options: {}", prune_options.cli_options_string()));
-            worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
-                               store, backup_type, backup_id));
-        }
+    if keep_all {
+        worker.log("No prune selection - keeping all files.");
+    } else {
+        worker.log(format!("retention options: {}", prune_options.cli_options_string()));
+        worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
+                            store, backup_type, backup_id));
+    }
 
-        for (info, mut keep) in prune_info {
-            if keep_all { keep = true; }
+    for (info, mut keep) in prune_info {
+        if keep_all { keep = true; }
 
-            let backup_time = info.backup_dir.backup_time();
-            let timestamp = info.backup_dir.backup_time_string();
-            let group = info.backup_dir.group();
+        let backup_time = info.backup_dir.backup_time();
+        let timestamp = info.backup_dir.backup_time_string();
+        let group = info.backup_dir.group();
 
 
-            let msg = format!(
-                "{}/{}/{} {}",
-                group.backup_type(),
-                group.backup_id(),
-                timestamp,
-                if keep { "keep" } else { "remove" },
-            );
+        let msg = format!(
+            "{}/{}/{} {}",
+            group.backup_type(),
+            group.backup_id(),
+            timestamp,
+            if keep { "keep" } else { "remove" },
+        );
 
-            worker.log(msg);
+        worker.log(msg);
 
-            prune_result.push(json!({
-                "backup-type": group.backup_type(),
-                "backup-id": group.backup_id(),
-                "backup-time": backup_time,
-                "keep": keep,
-            }));
+        prune_result.push(json!({
+            "backup-type": group.backup_type(),
+            "backup-id": group.backup_id(),
+            "backup-time": backup_time,
+            "keep": keep,
+        }));
 
-            if !(dry_run || keep) {
-                datastore.remove_backup_dir(&info.backup_dir, true)?;
+        if !(dry_run || keep) {
+            if let Err(err) = datastore.remove_backup_dir(&info.backup_dir, false) {
+                worker.warn(
+                    format!(
+                        "failed to remove dir {:?}: {}",
+                        info.backup_dir.relative_path(), err
+                    )
+                );
             }
         }
+    }
 
-        Ok(())
-    };
-
-    worker.log_result(&result);
-
-    if let Err(err) = result {
-        bail!("prune failed - {}", err);
-    };
+    worker.log_result(&Ok(()));
 
     Ok(json!(prune_result))
 }
@@ -770,7 +843,7 @@ fn start_garbage_collection(
         to_stdout,
         move |worker| {
             worker.log(format!("starting garbage collection on store {}", store));
-            datastore.garbage_collection(&worker)
+            datastore.garbage_collection(&*worker, worker.upid())
         },
     )?;
 
@@ -1235,6 +1308,66 @@ fn catalog(
     Ok(res.into())
 }
 
+fn recurse_files<'a, T, W>(
+    zip: &'a mut ZipEncoder<W>,
+    decoder: &'a mut Accessor<T>,
+    prefix: &'a Path,
+    file: FileEntry<T>,
+) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'a>>
+where
+    T: Clone + pxar::accessor::ReadAt + Unpin + Send + Sync + 'static,
+    W: tokio::io::AsyncWrite + Unpin + Send + 'static,
+{
+    Box::pin(async move {
+        let metadata = file.entry().metadata();
+        let path = file.entry().path().strip_prefix(&prefix)?.to_path_buf();
+
+        match file.kind() {
+            EntryKind::File { .. } => {
+                let entry = ZipEntry::new(
+                    path,
+                    metadata.stat.mtime.secs,
+                    metadata.stat.mode as u16,
+                    true,
+                );
+                zip.add_entry(entry, Some(file.contents().await?))
+                   .await
+                   .map_err(|err| format_err!("could not send file entry: {}", err))?;
+            }
+            EntryKind::Hardlink(_) => {
+                let realfile = decoder.follow_hardlink(&file).await?;
+                let entry = ZipEntry::new(
+                    path,
+                    metadata.stat.mtime.secs,
+                    metadata.stat.mode as u16,
+                    true,
+                );
+                zip.add_entry(entry, Some(realfile.contents().await?))
+                   .await
+                   .map_err(|err| format_err!("could not send file entry: {}", err))?;
+            }
+            EntryKind::Directory => {
+                let dir = file.enter_directory().await?;
+                let mut readdir = dir.read_dir();
+                let entry = ZipEntry::new(
+                    path,
+                    metadata.stat.mtime.secs,
+                    metadata.stat.mode as u16,
+                    false,
+                );
+                zip.add_entry::<FileContents<T>>(entry, None).await?;
+                while let Some(entry) = readdir.next().await {
+                    let entry = entry?.decode_entry().await?;
+                    recurse_files(zip, decoder, prefix, entry).await?;
+                }
+            }
+            _ => {} // ignore all else
+        };
+
+        Ok(())
+    })
+}
+
 #[sortable]
 pub const API_METHOD_PXAR_FILE_DOWNLOAD: ApiMethod = ApiMethod::new(
     &ApiHandler::AsyncHttp(&pxar_file_download),
@@ -1317,23 +1450,55 @@ fn pxar_file_download(
             .lookup(OsStr::from_bytes(file_path)).await?
             .ok_or(format_err!("error opening '{:?}'", file_path))?;
 
-        let file = match file.kind() {
-            EntryKind::File { .. } => file,
-            EntryKind::Hardlink(_) => {
-                decoder.follow_hardlink(&file).await?
-            },
-            // TODO symlink
+        let body = match file.kind() {
+            EntryKind::File { .. } => Body::wrap_stream(
+                AsyncReaderStream::new(file.contents().await?).map_err(move |err| {
+                    eprintln!("error during streaming of file '{:?}' - {}", filepath, err);
+                    err
+                }),
+            ),
+            EntryKind::Hardlink(_) => Body::wrap_stream(
+                AsyncReaderStream::new(decoder.follow_hardlink(&file).await?.contents().await?)
+                    .map_err(move |err| {
+                        eprintln!(
+                            "error during streaming of hardlink '{:?}' - {}",
+                            filepath, err
+                        );
+                        err
+                    }),
+            ),
+            EntryKind::Directory => {
+                let (sender, receiver) = tokio::sync::mpsc::channel(100);
+                let mut prefix = PathBuf::new();
+                let mut components = file.entry().path().components();
+                components.next_back(); // discar last
+                for comp in components {
+                    prefix.push(comp);
+                }
+
+                let channelwriter = AsyncChannelWriter::new(sender, 1024 * 1024);
+
+                crate::server::spawn_internal_task(async move {
+                    let mut zipencoder = ZipEncoder::new(channelwriter);
+                    let mut decoder = decoder;
+                    recurse_files(&mut zipencoder, &mut decoder, &prefix, file)
+                        .await
+                        .map_err(|err| eprintln!("error during creating of zip: {}", err))?;
+
+                    zipencoder
+                        .finish()
+                        .await
+                        .map_err(|err| eprintln!("error during finishing of zip: {}", err))
+                });
+
+                Body::wrap_stream(receiver.map_err(move |err| {
+                    eprintln!("error during streaming of zip '{:?}' - {}", filepath, err);
+                    err
+                }))
+            }
             other => bail!("cannot download file of type {:?}", other),
         };
 
-        let body = Body::wrap_stream(
-            AsyncReaderStream::new(file.contents().await?)
-                .map_err(move |err| {
-                    eprintln!("error during streaming of '{:?}' - {}", filepath, err);
-                    err
-                })
-        );
-
         // fixme: set other headers ?
         Ok(Response::builder()
            .status(StatusCode::OK)
@@ -1422,9 +1587,9 @@ fn get_notes(
     let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
     if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
 
-    let manifest = datastore.load_manifest_json(&backup_dir)?;
+    let (manifest, _) = datastore.load_manifest(&backup_dir)?;
 
-    let notes = manifest["unprotected"]["notes"]
+    let notes = manifest.unprotected["notes"]
         .as_str()
         .unwrap_or("");
 
@@ -1475,11 +1640,54 @@ fn set_notes(
     let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
     if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
 
-    let mut manifest = datastore.load_manifest_json(&backup_dir)?;
+    datastore.update_manifest(&backup_dir,|manifest| {
+        manifest.unprotected["notes"] = notes.into();
+    }).map_err(|err| format_err!("unable to update manifest blob - {}", err))?;
 
-    manifest["unprotected"]["notes"] = notes.into();
+    Ok(())
+}
 
-    datastore.store_manifest(&backup_dir, manifest)?;
+#[api(
+    input: {
+        properties: {
+            store: {
+                schema: DATASTORE_SCHEMA,
+            },
+            "backup-type": {
+                schema: BACKUP_TYPE_SCHEMA,
+            },
+            "backup-id": {
+                schema: BACKUP_ID_SCHEMA,
+            },
+            "new-owner": {
+                type: Userid,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY, true),
+    },
+)]
+/// Change owner of a backup group
+fn set_backup_owner(
+    store: String,
+    backup_type: String,
+    backup_id: String,
+    new_owner: Userid,
+    _rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+
+    let datastore = DataStore::lookup_datastore(&store)?;
+
+    let backup_group = BackupGroup::new(backup_type, backup_id);
+
+    let user_info = CachedUserInfo::new()?;
+
+    if !user_info.is_active_user(&new_owner) {
+        bail!("user '{}' is inactive or non-existent", new_owner);
+    }
+
+    datastore.set_owner(&backup_group, &new_owner, true)?;
 
     Ok(())
 }
@@ -1491,6 +1699,11 @@ const DATASTORE_INFO_SUBDIRS: SubdirMap = &[
         &Router::new()
             .get(&API_METHOD_CATALOG)
     ),
+    (
+        "change-owner",
+        &Router::new()
+            .post(&API_METHOD_SET_BACKUP_OWNER)
+    ),
     (
         "download",
         &Router::new()

src/api2/admin/sync.rs

@@ -9,13 +9,18 @@ use crate::api2::types::*;
 use crate::api2::pull::do_sync_job;
 use crate::config::sync::{self, SyncJobStatus, SyncJobConfig};
 use crate::server::UPID;
-use crate::config::jobstate::{Job, JobState};
+use crate::server::jobstate::{Job, JobState};
 use crate::tools::systemd::time::{
     parse_calendar_event, compute_next_event};
 
 #[api(
     input: {
-        properties: {},
+        properties: {
+            store: {
+                schema: DATASTORE_SCHEMA,
+                optional: true,
+            },
+        },
     },
     returns: {
         description: "List configured jobs and their status.",
@@ -25,13 +30,23 @@ use crate::tools::systemd::time::{
 )]
 /// List all sync jobs
 pub fn list_sync_jobs(
+    store: Option<String>,
     _param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<SyncJobStatus>, Error> {
 
     let (config, digest) = sync::config()?;
 
-    let mut list: Vec<SyncJobStatus> = config.convert_to_typed_array("sync")?;
+    let mut list: Vec<SyncJobStatus> = config
+        .convert_to_typed_array("sync")?
+        .into_iter()
+        .filter(|job: &SyncJobStatus| {
+            if let Some(store) = &store {
+                &job.store == store
+            } else {
+                true
+            }
+        }).collect();
 
     for job in &mut list {
         let last_state = JobState::load("syncjob", &job.id)

src/api2/admin/verify.rs

@@ -0,0 +1,122 @@
+use anyhow::{format_err, Error};
+
+use proxmox::api::router::SubdirMap;
+use proxmox::{list_subdirs_api_method, sortable};
+use proxmox::api::{api, ApiMethod, Router, RpcEnvironment};
+
+use crate::api2::types::*;
+use crate::server::do_verification_job;
+use crate::server::jobstate::{Job, JobState};
+use crate::config::verify;
+use crate::config::verify::{VerificationJobConfig, VerificationJobStatus};
+use serde_json::Value;
+use crate::tools::systemd::time::{parse_calendar_event, compute_next_event};
+use crate::server::UPID;
+
+#[api(
+    input: {
+        properties: {
+            store: {
+                schema: DATASTORE_SCHEMA,
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        description: "List configured jobs and their status.",
+        type: Array,
+        items: { type: verify::VerificationJobStatus },
+    },
+)]
+/// List all verification jobs
+pub fn list_verification_jobs(
+    store: Option<String>,
+    _param: Value,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<VerificationJobStatus>, Error> {
+
+    let (config, digest) = verify::config()?;
+
+    let mut list: Vec<VerificationJobStatus> = config
+        .convert_to_typed_array("verification")?
+        .into_iter()
+        .filter(|job: &VerificationJobStatus| {
+            if let Some(store) = &store {
+                &job.store == store
+            } else {
+                true
+            }
+        }).collect();
+
+    for job in &mut list {
+        let last_state = JobState::load("verificationjob", &job.id)
+            .map_err(|err| format_err!("could not open statefile for {}: {}", &job.id, err))?;
+
+        let (upid, endtime, state, starttime) = match last_state {
+            JobState::Created { time } => (None, None, None, time),
+            JobState::Started { upid } => {
+                let parsed_upid: UPID = upid.parse()?;
+                (Some(upid), None, None, parsed_upid.starttime)
+            },
+            JobState::Finished { upid, state } => {
+                let parsed_upid: UPID = upid.parse()?;
+                (Some(upid), Some(state.endtime()), Some(state.to_string()), parsed_upid.starttime)
+            },
+        };
+
+        job.last_run_upid = upid;
+        job.last_run_state = state;
+        job.last_run_endtime = endtime;
+
+        let last = job.last_run_endtime.unwrap_or_else(|| starttime);
+
+        job.next_run = (|| -> Option<i64> {
+            let schedule = job.schedule.as_ref()?;
+            let event = parse_calendar_event(&schedule).ok()?;
+            // ignore errors
+            compute_next_event(&event, last, false).unwrap_or_else(|_| None)
+        })();
+    }
+
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+    Ok(list)
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            }
+        }
+    }
+)]
+/// Runs a verification job manually.
+fn run_verification_job(
+    id: String,
+    _info: &ApiMethod,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+    let (config, _digest) = verify::config()?;
+    let verification_job: VerificationJobConfig = config.lookup("verification", &id)?;
+
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+
+    let job = Job::new("verificationjob", &id)?;
+
+    let upid_str = do_verification_job(job, verification_job, &userid, None)?;
+
+    Ok(upid_str)
+}
+
+#[sortable]
+const VERIFICATION_INFO_SUBDIRS: SubdirMap = &[("run", &Router::new().post(&API_METHOD_RUN_VERIFICATION_JOB))];
+
+const VERIFICATION_INFO_ROUTER: Router = Router::new()
+    .get(&list_subdirs_api_method!(VERIFICATION_INFO_SUBDIRS))
+    .subdirs(VERIFICATION_INFO_SUBDIRS);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_VERIFICATION_JOBS)
+    .match_all("id", &VERIFICATION_INFO_ROUTER);

src/api2/backup.rs

@@ -16,7 +16,7 @@ use crate::backup::*;
 use crate::api2::types::*;
 use crate::config::acl::PRIV_DATASTORE_BACKUP;
 use crate::config::cached_user_info::CachedUserInfo;
-use crate::tools::fs::lock_dir_noblock;
+use crate::tools::fs::lock_dir_noblock_shared;
 
 mod environment;
 use environment::*;
@@ -86,7 +86,7 @@ async move {
         bail!("unexpected http version '{:?}' (expected version < 2)", parts.version);
     }
 
-    let worker_id = format!("{}_{}_{}", store, backup_type, backup_id);
+    let worker_id = format!("{}:{}/{}", store, backup_type, backup_id);
 
     let env_type = rpcenv.env_type();
 
@@ -144,12 +144,12 @@ async move {
 
         // lock last snapshot to prevent forgetting/pruning it during backup
         let full_path = datastore.snapshot_path(&last.backup_dir);
-        Some(lock_dir_noblock(&full_path, "snapshot", "base snapshot is already locked by another operation")?)
+        Some(lock_dir_noblock_shared(&full_path, "snapshot", "base snapshot is already locked by another operation")?)
     } else {
         None
     };
 
-    let (path, is_new, _snap_guard) = datastore.create_locked_backup_dir(&backup_dir)?;
+    let (path, is_new, snap_guard) = datastore.create_locked_backup_dir(&backup_dir)?;
     if !is_new { bail!("backup directory already exists."); }
 
 
@@ -182,8 +182,22 @@ async move {
                 http.http2_initial_connection_window_size(window_size);
                 http.http2_max_frame_size(4*1024*1024);
 
+                let env3 = env2.clone();
                 http.serve_connection(conn, service)
-                    .map_err(Error::from)
+                    .map(move |result| {
+                        match result {
+                            Err(err) => {
+                                // Avoid  Transport endpoint is not connected (os error 107)
+                                // fixme: find a better way to test for that error
+                                if err.to_string().starts_with("connection error") && env3.finished() {
+                                    Ok(())
+                                } else {
+                                    Err(Error::from(err))
+                                }
+                            }
+                            Ok(()) => Ok(()),
+                        }
+                    })
             });
         let mut abort_future = abort_future
             .map(|_| Err(format_err!("task aborted")));
@@ -191,7 +205,7 @@ async move {
         async move {
             // keep flock until task ends
             let _group_guard = _group_guard;
-            let _snap_guard = _snap_guard;
+            let snap_guard = snap_guard;
             let _last_guard = _last_guard;
 
             let res = select!{
@@ -203,20 +217,32 @@ async move {
                 tools::runtime::block_in_place(|| env.remove_backup())?;
                 return Ok(());
             }
+
+            let verify = |env: BackupEnvironment| {
+                if let Err(err) = env.verify_after_complete(snap_guard) {
+                    env.log(format!(
+                        "backup finished, but starting the requested verify task failed: {}",
+                        err
+                    ));
+                }
+            };
+
             match (res, env.ensure_finished()) {
                 (Ok(_), Ok(())) => {
                     env.log("backup finished successfully");
+                    verify(env);
                     Ok(())
                 },
                 (Err(err), Ok(())) => {
                     // ignore errors after finish
                     env.log(format!("backup had errors but finished: {}", err));
+                    verify(env);
                     Ok(())
                 },
                 (Ok(_), Err(err)) => {
                     env.log(format!("backup ended and finish failed: {}", err));
                     env.log("removing unfinished backup");
-                    env.remove_backup()?;
+                    tools::runtime::block_in_place(|| env.remove_backup())?;
                     Err(err)
                 },
                 (Err(err), Err(_)) => {

src/api2/backup/environment.rs

@@ -1,6 +1,7 @@
 use anyhow::{bail, format_err, Error};
 use std::sync::{Arc, Mutex};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
+use nix::dir::Dir;
 
 use ::serde::{Serialize};
 use serde_json::{json, Value};
@@ -472,16 +473,11 @@ impl BackupEnvironment {
             bail!("backup does not contain valid files (file count == 0)");
         }
 
-        // check manifest
-        let mut manifest = self.datastore.load_manifest_json(&self.backup_dir)
-            .map_err(|err| format_err!("unable to load manifest blob - {}", err))?;
-
+        // check for valid manifest and store stats
         let stats = serde_json::to_value(state.backup_stat)?;
-
-        manifest["unprotected"]["chunk_upload_stats"] = stats;
-
-        self.datastore.store_manifest(&self.backup_dir, manifest)
-            .map_err(|err| format_err!("unable to store manifest blob - {}", err))?;
+        self.datastore.update_manifest(&self.backup_dir, |manifest| {
+            manifest.unprotected["chunk_upload_stats"] = stats;
+        }).map_err(|err| format_err!("unable to update manifest blob - {}", err))?;
 
         if let Some(base) = &self.last_backup {
             let path = self.datastore.snapshot_path(&base.backup_dir);
@@ -499,6 +495,54 @@ impl BackupEnvironment {
         Ok(())
     }
 
+    /// If verify-new is set on the datastore, this will run a new verify task
+    /// for the backup. If not, this will return and also drop the passed lock
+    /// immediately.
+    pub fn verify_after_complete(&self, snap_lock: Dir) -> Result<(), Error> {
+        self.ensure_finished()?;
+
+        if !self.datastore.verify_new() {
+            // no verify requested, do nothing
+            return Ok(());
+        }
+
+        let worker_id = format!("{}:{}/{}/{:08X}",
+            self.datastore.name(),
+            self.backup_dir.group().backup_type(),
+            self.backup_dir.group().backup_id(),
+            self.backup_dir.backup_time());
+
+        let datastore = self.datastore.clone();
+        let backup_dir = self.backup_dir.clone();
+
+        WorkerTask::new_thread(
+            "verify",
+            Some(worker_id),
+            self.user.clone(),
+            false,
+            move |worker| {
+                worker.log("Automatically verifying newly added snapshot");
+
+                let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*16)));
+                let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
+
+                if !verify_backup_dir_with_lock(
+                    datastore,
+                    &backup_dir,
+                    verified_chunks,
+                    corrupt_chunks,
+                    worker.clone(),
+                    worker.upid().clone(),
+                    snap_lock,
+                )? {
+                    bail!("verification failed - please check the log for details");
+                }
+
+                Ok(())
+            },
+        ).map(|_| ())
+    }
+
     pub fn log<S: AsRef<str>>(&self, msg: S) {
         self.worker.log(msg);
     }
@@ -523,6 +567,12 @@ impl BackupEnvironment {
         Ok(())
     }
 
+    /// Return true if the finished flag is set
+    pub fn finished(&self) -> bool {
+        let state = self.state.lock().unwrap();
+        state.finished
+    }
+
     /// Remove complete backup
     pub fn remove_backup(&self) -> Result<(), Error> {
         let mut state = self.state.lock().unwrap();

src/api2/config.rs

@@ -4,11 +4,13 @@ use proxmox::list_subdirs_api_method;
 pub mod datastore;
 pub mod remote;
 pub mod sync;
+pub mod verify;
 
 const SUBDIRS: SubdirMap = &[
     ("datastore", &datastore::ROUTER),
     ("remote", &remote::ROUTER),
     ("sync", &sync::ROUTER),
+    ("verify", &verify::ROUTER)
 ];
 
 pub const ROUTER: Router = Router::new()

src/api2/config/datastore.rs

@@ -11,7 +11,8 @@ use crate::api2::types::*;
 use crate::backup::*;
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::datastore::{self, DataStoreConfig, DIR_NAME_SCHEMA};
-use crate::config::acl::{PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY};
+use crate::config::acl::{PRIV_DATASTORE_ALLOCATE, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY};
+use crate::server::jobstate;
 
 #[api(
     input: {
@@ -75,10 +76,6 @@ pub fn list_datastores(
                 optional: true,
                 schema: PRUNE_SCHEDULE_SCHEMA,
             },
-            "verify-schedule": {
-                optional: true,
-                schema: VERIFY_SCHEDULE_SCHEMA,
-            },
             "keep-last": {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -106,7 +103,7 @@ pub fn list_datastores(
         },
     },
     access: {
-        permission: &Permission::Privilege(&["datastore"], PRIV_DATASTORE_MODIFY, false),
+        permission: &Permission::Privilege(&["datastore"], PRIV_DATASTORE_ALLOCATE, false),
     },
 )]
 /// Create new datastore config.
@@ -131,9 +128,8 @@ pub fn create_datastore(param: Value) -> Result<(), Error> {
 
     datastore::save_config(&config)?;
 
-    crate::config::jobstate::create_state_file("prune", &datastore.name)?;
-    crate::config::jobstate::create_state_file("garbage_collection", &datastore.name)?;
-    crate::config::jobstate::create_state_file("verify", &datastore.name)?;
+    jobstate::create_state_file("prune", &datastore.name)?;
+    jobstate::create_state_file("garbage_collection", &datastore.name)?;
 
     Ok(())
 }
@@ -179,8 +175,6 @@ pub enum DeletableProperty {
     gc_schedule,
     /// Delete the prune job schedule.
     prune_schedule,
-    /// Delete the verify schedule property
-    verify_schedule,
     /// Delete the keep-last property
     keep_last,
     /// Delete the keep-hourly property
@@ -214,10 +208,6 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: PRUNE_SCHEDULE_SCHEMA,
             },
-            "verify-schedule": {
-                optional: true,
-                schema: VERIFY_SCHEDULE_SCHEMA,
-            },
             "keep-last": {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -266,7 +256,6 @@ pub fn update_datastore(
     comment: Option<String>,
     gc_schedule: Option<String>,
     prune_schedule: Option<String>,
-    verify_schedule: Option<String>,
     keep_last: Option<u64>,
     keep_hourly: Option<u64>,
     keep_daily: Option<u64>,
@@ -295,7 +284,6 @@ pub fn update_datastore(
                 DeletableProperty::comment => { data.comment = None; },
                 DeletableProperty::gc_schedule => { data.gc_schedule = None; },
                 DeletableProperty::prune_schedule => { data.prune_schedule = None; },
-                DeletableProperty::verify_schedule => { data.verify_schedule = None; },
                 DeletableProperty::keep_last => { data.keep_last = None; },
                 DeletableProperty::keep_hourly => { data.keep_hourly = None; },
                 DeletableProperty::keep_daily => { data.keep_daily = None; },
@@ -327,12 +315,6 @@ pub fn update_datastore(
         data.prune_schedule = prune_schedule;
     }
 
-    let mut verify_schedule_changed = false;
-    if verify_schedule.is_some() {
-        verify_schedule_changed = data.verify_schedule != verify_schedule;
-        data.verify_schedule = verify_schedule;
-    }
-
     if keep_last.is_some() { data.keep_last = keep_last; }
     if keep_hourly.is_some() { data.keep_hourly = keep_hourly; }
     if keep_daily.is_some() { data.keep_daily = keep_daily; }
@@ -347,15 +329,11 @@ pub fn update_datastore(
     // we want to reset the statefiles, to avoid an immediate action in some cases
     // (e.g. going from monthly to weekly in the second week of the month)
     if gc_schedule_changed {
-        crate::config::jobstate::create_state_file("garbage_collection", &name)?;
+        jobstate::create_state_file("garbage_collection", &name)?;
     }
 
     if prune_schedule_changed {
-        crate::config::jobstate::create_state_file("prune", &name)?;
-    }
-
-    if verify_schedule_changed {
-        crate::config::jobstate::create_state_file("verify", &name)?;
+        jobstate::create_state_file("prune", &name)?;
     }
 
     Ok(())
@@ -375,7 +353,7 @@ pub fn update_datastore(
         },
     },
     access: {
-        permission: &Permission::Privilege(&["datastore", "{name}"], PRIV_DATASTORE_MODIFY, false),
+        permission: &Permission::Privilege(&["datastore", "{name}"], PRIV_DATASTORE_ALLOCATE, false),
     },
 )]
 /// Remove a datastore configuration.
@@ -398,9 +376,8 @@ pub fn delete_datastore(name: String, digest: Option<String>) -> Result<(), Erro
     datastore::save_config(&config)?;
 
     // ignore errors
-    let _ = crate::config::jobstate::remove_state_file("prune", &name);
-    let _ = crate::config::jobstate::remove_state_file("garbage_collection", &name);
-    let _ = crate::config::jobstate::remove_state_file("verify", &name);
+    let _ = jobstate::remove_state_file("prune", &name);
+    let _ = jobstate::remove_state_file("garbage_collection", &name);
 
     Ok(())
 }

src/api2/config/remote.rs

@@ -1,7 +1,6 @@
 use anyhow::{bail, Error};
 use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
-use base64;
 
 use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
 use proxmox::tools::fs::open_file_locked;

src/api2/config/sync.rs

@@ -83,7 +83,7 @@ pub fn create_sync_job(param: Value) -> Result<(), Error> {
 
     sync::save_config(&config)?;
 
-    crate::config::jobstate::create_state_file("syncjob", &sync_job.id)?;
+    crate::server::jobstate::create_state_file("syncjob", &sync_job.id)?;
 
     Ok(())
 }
@@ -266,7 +266,7 @@ pub fn delete_sync_job(id: String, digest: Option<String>) -> Result<(), Error>
 
     sync::save_config(&config)?;
 
-    crate::config::jobstate::remove_state_file("syncjob", &id)?;
+    crate::server::jobstate::remove_state_file("syncjob", &id)?;
 
     Ok(())
 }

src/api2/config/verify.rs

@@ -0,0 +1,274 @@
+use anyhow::{bail, Error};
+use serde_json::Value;
+use ::serde::{Deserialize, Serialize};
+
+use proxmox::api::{api, Router, RpcEnvironment};
+use proxmox::tools::fs::open_file_locked;
+
+use crate::api2::types::*;
+use crate::config::verify::{self, VerificationJobConfig};
+
+#[api(
+    input: {
+        properties: {},
+    },
+    returns: {
+        description: "List configured jobs.",
+        type: Array,
+        items: { type: verify::VerificationJobConfig },
+    },
+)]
+/// List all verification jobs
+pub fn list_verification_jobs(
+    _param: Value,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<VerificationJobConfig>, Error> {
+
+    let (config, digest) = verify::config()?;
+
+    let list = config.convert_to_typed_array("verification")?;
+
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+    Ok(list)
+}
+
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            store: {
+                schema: DATASTORE_SCHEMA,
+            },
+            "ignore-verified": {
+                optional: true,
+                schema: IGNORE_VERIFIED_BACKUPS_SCHEMA,
+            },
+            "outdated-after": {
+                optional: true,
+                schema: VERIFICATION_OUTDATED_AFTER_SCHEMA,
+            },
+            comment: {
+                optional: true,
+                schema: SINGLE_LINE_COMMENT_SCHEMA,
+            },
+            schedule: {
+                optional: true,
+                schema: VERIFICATION_SCHEDULE_SCHEMA,
+            },
+        }
+    }
+)]
+/// Create a new verification job.
+pub fn create_verification_job(param: Value) -> Result<(), Error> {
+
+    let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
+    let verification_job: verify::VerificationJobConfig = serde_json::from_value(param.clone())?;
+
+    let (mut config, _digest) = verify::config()?;
+
+    if let Some(_) = config.sections.get(&verification_job.id) {
+        bail!("job '{}' already exists.", verification_job.id);
+    }
+
+    config.set_data(&verification_job.id, "verification", &verification_job)?;
+
+    verify::save_config(&config)?;
+
+    crate::server::jobstate::create_state_file("verificationjob", &verification_job.id)?;
+
+    Ok(())
+}
+
+#[api(
+   input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+        },
+    },
+    returns: {
+        description: "The verification job configuration.",
+        type: verify::VerificationJobConfig,
+    },
+)]
+/// Read a verification job configuration.
+pub fn read_verification_job(
+    id: String,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<VerificationJobConfig, Error> {
+    let (config, digest) = verify::config()?;
+
+    let verification_job = config.lookup("verification", &id)?;
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+    Ok(verification_job)
+}
+
+#[api()]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all="kebab-case")]
+/// Deletable property name
+pub enum DeletableProperty {
+    /// Delete the ignore verified property.
+    IgnoreVerified,
+    /// Delete the comment property.
+    Comment,
+    /// Delete the job schedule.
+    Schedule,
+    /// Delete outdated after property.
+    OutdatedAfter
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            store: {
+                optional: true,
+                schema: DATASTORE_SCHEMA,
+            },
+            "ignore-verified": {
+                optional: true,
+                schema: IGNORE_VERIFIED_BACKUPS_SCHEMA,
+            },
+            "outdated-after": {
+                optional: true,
+                schema: VERIFICATION_OUTDATED_AFTER_SCHEMA,
+            },
+            comment: {
+                optional: true,
+                schema: SINGLE_LINE_COMMENT_SCHEMA,
+            },
+            schedule: {
+                optional: true,
+                schema: VERIFICATION_SCHEDULE_SCHEMA,
+            },
+            delete: {
+                description: "List of properties to delete.",
+                type: Array,
+                optional: true,
+                items: {
+                    type: DeletableProperty,
+                }
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+)]
+/// Update verification job config.
+pub fn update_verification_job(
+    id: String,
+    store: Option<String>,
+    ignore_verified: Option<bool>,
+    outdated_after: Option<i64>,
+    comment: Option<String>,
+    schedule: Option<String>,
+    delete: Option<Vec<DeletableProperty>>,
+    digest: Option<String>,
+) -> Result<(), Error> {
+
+    let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
+    // pass/compare digest
+    let (mut config, expected_digest) = verify::config()?;
+
+    if let Some(ref digest) = digest {
+        let digest = proxmox::tools::hex_to_digest(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    let mut data: verify::VerificationJobConfig = config.lookup("verification", &id)?;
+
+     if let Some(delete) = delete {
+        for delete_prop in delete {
+            match delete_prop {
+                DeletableProperty::IgnoreVerified => { data.ignore_verified = None; },
+                DeletableProperty::OutdatedAfter => { data.outdated_after = None; },
+                DeletableProperty::Comment => { data.comment = None; },
+                DeletableProperty::Schedule => { data.schedule = None; },
+            }
+        }
+    }
+
+    if let Some(comment) = comment {
+        let comment = comment.trim().to_string();
+        if comment.is_empty() {
+            data.comment = None;
+        } else {
+            data.comment = Some(comment);
+        }
+    }
+
+    if let Some(store) = store { data.store = store; }
+
+    if ignore_verified.is_some() { data.ignore_verified = ignore_verified; }
+    if outdated_after.is_some() { data.outdated_after = outdated_after; }
+    if schedule.is_some() { data.schedule = schedule; }
+
+    config.set_data(&id, "verification", &data)?;
+
+    verify::save_config(&config)?;
+
+    Ok(())
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+)]
+/// Remove a verification job configuration
+pub fn delete_verification_job(id: String, digest: Option<String>) -> Result<(), Error> {
+
+    let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
+    let (mut config, expected_digest) = verify::config()?;
+
+    if let Some(ref digest) = digest {
+        let digest = proxmox::tools::hex_to_digest(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    match config.sections.get(&id) {
+        Some(_) => { config.sections.remove(&id); },
+        None => bail!("job '{}' does not exist.", id),
+    }
+
+    verify::save_config(&config)?;
+
+    crate::server::jobstate::remove_state_file("verificationjob", &id)?;
+
+    Ok(())
+}
+
+const ITEM_ROUTER: Router = Router::new()
+    .get(&API_METHOD_READ_VERIFICATION_JOB)
+    .put(&API_METHOD_UPDATE_VERIFICATION_JOB)
+    .delete(&API_METHOD_DELETE_VERIFICATION_JOB);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_VERIFICATION_JOBS)
+    .post(&API_METHOD_CREATE_VERIFICATION_JOB)
+    .match_all("id", &ITEM_ROUTER);

src/api2/node/apt.rs

@@ -1,5 +1,7 @@
+use std::collections::HashSet;
+
 use apt_pkg_native::Cache;
-use anyhow::{Error, bail};
+use anyhow::{Error, bail, format_err};
 use serde_json::{json, Value};
 
 use proxmox::{list_subdirs_api_method, const_regex};
@@ -7,6 +9,7 @@ use proxmox::api::{api, RpcEnvironment, RpcEnvironmentType, Permission};
 use proxmox::api::router::{Router, SubdirMap};
 
 use crate::server::WorkerTask;
+use crate::tools::http;
 
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
 use crate::api2::types::{APTUpdateInfo, NODE_SCHEMA, Userid, UPID_SCHEMA};
@@ -16,14 +19,13 @@ const_regex! {
     FILENAME_EXTRACT_REGEX = r"^.*/.*?_(.*)_Packages$";
 }
 
-// FIXME: Replace with call to 'apt changelog <pkg> --print-uris'. Currently
-// not possible as our packages do not have a URI set in their Release file
+// FIXME: once the 'changelog' API call switches over to 'apt-get changelog' only,
+// consider removing this function entirely, as it's value is never used anywhere
+// then (widget-toolkit doesn't use the value either)
 fn get_changelog_url(
     package: &str,
     filename: &str,
-    source_pkg: &str,
     version: &str,
-    source_version: &str,
     origin: &str,
     component: &str,
 ) -> Result<String, Error> {
@@ -32,25 +34,24 @@ fn get_changelog_url(
     }
 
     if origin == "Debian" {
-        let source_version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(source_version, "");
-
-        let prefix = if source_pkg.starts_with("lib") {
-            source_pkg.get(0..4)
-        } else {
-            source_pkg.get(0..1)
+        let mut command = std::process::Command::new("apt-get");
+        command.arg("changelog");
+        command.arg("--print-uris");
+        command.arg(package);
+        let output = crate::tools::run_command(command, None)?; // format: 'http://foo/bar' package.changelog
+        let output = match output.splitn(2, ' ').next() {
+            Some(output) => {
+                if output.len() < 2 {
+                    bail!("invalid output (URI part too short) from 'apt-get changelog --print-uris': {}", output)
+                }
+                output[1..output.len()-1].to_owned()
+            },
+            None => bail!("invalid output from 'apt-get changelog --print-uris': {}", output)
         };
-
-        let prefix = match prefix {
-            Some(p) => p,
-            None => bail!("cannot get starting characters of package name '{}'", package)
-        };
-
-        // note: security updates seem to not always upload a changelog for
-        // their package version, so this only works *most* of the time
-        return Ok(format!("https://metadata.ftp-master.debian.org/changelogs/main/{}/{}/{}_{}_changelog",
-                          prefix, source_pkg, source_pkg, source_version));
-
+        return Ok(output);
     } else if origin == "Proxmox" {
+        // FIXME: Use above call to 'apt changelog <pkg> --print-uris' as well.
+        // Currently not possible as our packages do not have a URI set in their Release file.
         let version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(version, "");
 
         let base = match (FILENAME_EXTRACT_REGEX.regex_obj)().captures(filename) {
@@ -71,115 +72,229 @@ fn get_changelog_url(
     bail!("unknown origin ({}) or component ({})", origin, component)
 }
 
-fn list_installed_apt_packages<F: Fn(&str, &str, &str) -> bool>(filter: F)
-    -> Vec<APTUpdateInfo> {
+struct FilterData<'a> {
+    // this is version info returned by APT
+    installed_version: Option<&'a str>,
+    candidate_version: &'a str,
+
+    // this is the version info the filter is supposed to check
+    active_version: &'a str,
+}
+
+enum PackagePreSelect {
+    OnlyInstalled,
+    OnlyNew,
+    All,
+}
+
+fn list_installed_apt_packages<F: Fn(FilterData) -> bool>(
+    filter: F,
+    only_versions_for: Option<&str>,
+) -> Vec<APTUpdateInfo> {
 
     let mut ret = Vec::new();
+    let mut depends = HashSet::new();
 
     // note: this is not an 'apt update', it just re-reads the cache from disk
     let mut cache = Cache::get_singleton();
     cache.reload();
 
-    let mut cache_iter = cache.iter();
+    let mut cache_iter = match only_versions_for {
+        Some(name) => cache.find_by_name(name),
+        None => cache.iter()
+    };
 
     loop {
-        let view = match cache_iter.next() {
-            Some(view) => view,
-            None => break
-        };
 
-        let current_version = match view.current_version() {
-            Some(vers) => vers,
-            None => continue
-        };
-        let candidate_version = match view.candidate_version() {
-            Some(vers) => vers,
-            // if there's no candidate (i.e. no update) get info of currently
-            // installed version instead
-            None => current_version.clone()
-        };
+        match cache_iter.next() {
+            Some(view) => {
+                let di = if only_versions_for.is_some() {
+                    query_detailed_info(
+                        PackagePreSelect::All,
+                        &filter,
+                        view,
+                        None
+                    )
+                } else {
+                    query_detailed_info(
+                        PackagePreSelect::OnlyInstalled,
+                        &filter,
+                        view,
+                        Some(&mut depends)
+                    )
+                };
+                if let Some(info) = di {
+                    ret.push(info);
+                }
+
+                if only_versions_for.is_some() {
+                    break;
+                }
+            },
+            None => {
+                drop(cache_iter);
+                // also loop through missing dependencies, as they would be installed
+                for pkg in depends.iter() {
+                    let mut iter = cache.find_by_name(&pkg);
+                    let view = match iter.next() {
+                        Some(view) => view,
+                        None => continue // package not found, ignore
+                    };
+
+                    let di = query_detailed_info(
+                        PackagePreSelect::OnlyNew,
+                        &filter,
+                        view,
+                        None
+                    );
+                    if let Some(info) = di {
+                        ret.push(info);
+                    }
+                }
+                break;
+            }
+        }
+    }
+
+    return ret;
+}
+
+fn query_detailed_info<'a, F, V>(
+    pre_select: PackagePreSelect,
+    filter: F,
+    view: V,
+    depends: Option<&mut HashSet<String>>,
+) -> Option<APTUpdateInfo>
+where
+    F: Fn(FilterData) -> bool,
+    V: std::ops::Deref<Target = apt_pkg_native::sane::PkgView<'a>>
+{
+    let current_version = view.current_version();
+    let candidate_version = view.candidate_version();
+
+    let (current_version, candidate_version) = match pre_select {
+        PackagePreSelect::OnlyInstalled => match (current_version, candidate_version) {
+            (Some(cur), Some(can)) => (Some(cur), can), // package installed and there is an update
+            (Some(cur), None) => (Some(cur.clone()), cur), // package installed and up-to-date
+            (None, Some(_)) => return None, // package could be installed
+            (None, None) => return None, // broken
+        },
+        PackagePreSelect::OnlyNew => match (current_version, candidate_version) {
+            (Some(_), Some(_)) => return None,
+            (Some(_), None) => return None,
+            (None, Some(can)) => (None, can),
+            (None, None) => return None,
+        },
+        PackagePreSelect::All => match (current_version, candidate_version) {
+            (Some(cur), Some(can)) => (Some(cur), can),
+            (Some(cur), None) => (Some(cur.clone()), cur),
+            (None, Some(can)) => (None, can),
+            (None, None) => return None,
+        },
+    };
+
+    // get additional information via nested APT 'iterators'
+    let mut view_iter = view.versions();
+    while let Some(ver) = view_iter.next() {
 
         let package = view.name();
-        if filter(&package, &current_version, &candidate_version) {
-            let mut origin_res = "unknown".to_owned();
-            let mut section_res = "unknown".to_owned();
-            let mut priority_res = "unknown".to_owned();
-            let mut change_log_url = "".to_owned();
-            let mut short_desc = package.clone();
-            let mut long_desc = "".to_owned();
+        let version = ver.version();
+        let mut origin_res = "unknown".to_owned();
+        let mut section_res = "unknown".to_owned();
+        let mut priority_res = "unknown".to_owned();
+        let mut change_log_url = "".to_owned();
+        let mut short_desc = package.clone();
+        let mut long_desc = "".to_owned();
 
-            // get additional information via nested APT 'iterators'
-            let mut view_iter = view.versions();
-            while let Some(ver) = view_iter.next() {
-                if ver.version() == candidate_version {
-                    if let Some(section) = ver.section() {
-                        section_res = section;
+        let fd = FilterData {
+            installed_version: current_version.as_deref(),
+            candidate_version: &candidate_version,
+            active_version: &version,
+        };
+
+        if filter(fd) {
+            if let Some(section) = ver.section() {
+                section_res = section;
+            }
+
+            if let Some(prio) = ver.priority_type() {
+                priority_res = prio;
+            }
+
+            // assume every package has only one origin file (not
+            // origin, but origin *file*, for some reason those seem to
+            // be different concepts in APT)
+            let mut origin_iter = ver.origin_iter();
+            let origin = origin_iter.next();
+            if let Some(origin) = origin {
+
+                if let Some(sd) = origin.short_desc() {
+                    short_desc = sd;
+                }
+
+                if let Some(ld) = origin.long_desc() {
+                    long_desc = ld;
+                }
+
+                // the package files appear in priority order, meaning
+                // the one for the candidate version is first - this is fine
+                // however, as the source package should be the same for all
+                // versions anyway
+                let mut pkg_iter = origin.file();
+                let pkg_file = pkg_iter.next();
+                if let Some(pkg_file) = pkg_file {
+                    if let Some(origin_name) = pkg_file.origin() {
+                        origin_res = origin_name;
                     }
 
-                    if let Some(prio) = ver.priority_type() {
-                        priority_res = prio;
+                    let filename = pkg_file.file_name();
+                    let component = pkg_file.component();
+
+                    // build changelog URL from gathered information
+                    // ignore errors, use empty changelog instead
+                    let url = get_changelog_url(&package, &filename,
+                        &version, &origin_res, &component);
+                    if let Ok(url) = url {
+                        change_log_url = url;
                     }
-
-                    // assume every package has only one origin file (not
-                    // origin, but origin *file*, for some reason those seem to
-                    // be different concepts in APT)
-                    let mut origin_iter = ver.origin_iter();
-                    let origin = origin_iter.next();
-                    if let Some(origin) = origin {
-
-                        if let Some(sd) = origin.short_desc() {
-                            short_desc = sd;
-                        }
-
-                        if let Some(ld) = origin.long_desc() {
-                            long_desc = ld;
-                        }
-
-                        // the package files appear in priority order, meaning
-                        // the one for the candidate version is first
-                        let mut pkg_iter = origin.file();
-                        let pkg_file = pkg_iter.next();
-                        if let Some(pkg_file) = pkg_file {
-                            if let Some(origin_name) = pkg_file.origin() {
-                                origin_res = origin_name;
-                            }
-
-                            let filename = pkg_file.file_name();
-                            let source_pkg = ver.source_package();
-                            let source_ver = ver.source_version();
-                            let component = pkg_file.component();
-
-                            // build changelog URL from gathered information
-                            // ignore errors, use empty changelog instead
-                            let url = get_changelog_url(&package, &filename, &source_pkg,
-                                &candidate_version, &source_ver, &origin_res, &component);
-                            if let Ok(url) = url {
-                                change_log_url = url;
-                            }
-                        }
-                    }
-
-                    break;
                 }
             }
 
-            let info = APTUpdateInfo {
+            if let Some(depends) = depends {
+                let mut dep_iter = ver.dep_iter();
+                loop {
+                    let dep = match dep_iter.next() {
+                        Some(dep) if dep.dep_type() != "Depends" => continue,
+                        Some(dep) => dep,
+                        None => break
+                    };
+
+                    let dep_pkg = dep.target_pkg();
+                    let name = dep_pkg.name();
+
+                    depends.insert(name);
+                }
+            }
+
+            return Some(APTUpdateInfo {
                 package,
                 title: short_desc,
                 arch: view.arch(),
                 description: long_desc,
                 change_log_url,
                 origin: origin_res,
-                version: candidate_version,
-                old_version: current_version,
+                version: candidate_version.clone(),
+                old_version: match current_version {
+                    Some(vers) => vers,
+                    None => "".to_owned()
+                },
                 priority: priority_res,
                 section: section_res,
-            };
-            ret.push(info);
+            });
         }
     }
 
-    return ret;
+    return None;
 }
 
 #[api(
@@ -201,8 +316,11 @@ fn list_installed_apt_packages<F: Fn(&str, &str, &str) -> bool>(filter: F)
 )]
 /// List available APT updates
 fn apt_update_available(_param: Value) -> Result<Value, Error> {
-    let ret = list_installed_apt_packages(|_pkg, cur_ver, can_ver| cur_ver != can_ver);
-    Ok(json!(ret))
+    let all_upgradeable = list_installed_apt_packages(|data| {
+        data.candidate_version == data.active_version &&
+        data.installed_version != Some(data.candidate_version)
+    }, None);
+    Ok(json!(all_upgradeable))
 }
 
 #[api(
@@ -256,7 +374,67 @@ pub fn apt_update_database(
     Ok(upid_str)
 }
 
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            name: {
+                description: "Package name to get changelog of.",
+                type: String,
+            },
+            version: {
+                description: "Package version to get changelog of. Omit to use candidate version.",
+                type: String,
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        schema: UPID_SCHEMA,
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Retrieve the changelog of the specified package.
+fn apt_get_changelog(
+    param: Value,
+) -> Result<Value, Error> {
+
+    let name = crate::tools::required_string_param(&param, "name")?.to_owned();
+    let version = param["version"].as_str();
+
+    let pkg_info = list_installed_apt_packages(|data| {
+        match version {
+            Some(version) => version == data.active_version,
+            None => data.active_version == data.candidate_version
+        }
+    }, Some(&name));
+
+    if pkg_info.len() == 0 {
+        bail!("Package '{}' not found", name);
+    }
+
+    let changelog_url = &pkg_info[0].change_log_url;
+    // FIXME: use 'apt-get changelog' for proxmox packages as well, once repo supports it
+    if changelog_url.starts_with("http://download.proxmox.com/") {
+        let changelog = crate::tools::runtime::block_on(http::get_string(changelog_url))
+            .map_err(|err| format_err!("Error downloading changelog from '{}': {}", changelog_url, err))?;
+        return Ok(json!(changelog));
+    } else {
+        let mut command = std::process::Command::new("apt-get");
+        command.arg("changelog");
+        command.arg("-qq"); // don't display download progress
+        command.arg(name);
+        let output = crate::tools::run_command(command, None)?;
+        return Ok(json!(output));
+    }
+}
+
 const SUBDIRS: SubdirMap = &[
+    ("changelog", &Router::new().get(&API_METHOD_APT_GET_CHANGELOG)),
     ("update", &Router::new()
         .get(&API_METHOD_APT_UPDATE_AVAILABLE)
         .post(&API_METHOD_APT_UPDATE_DATABASE)

src/api2/node/disks/zfs.rs

@@ -1,6 +1,6 @@
 use anyhow::{bail, Error};
 use serde_json::{json, Value};
-use ::serde::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize};
 
 use proxmox::api::{
     api, Permission, RpcEnvironment, RpcEnvironmentType,

src/api2/node/rrd.rs

@@ -31,10 +31,8 @@ pub fn create_value_from_rrd(
                 } else {
                     result.push(json!({ "time": t }));
                 }
-            } else {
-                if let Some(value) = list[index] {
-                    result[index][name] = value.into();
-                }
+            } else if let Some(value) = list[index] {
+                result[index][name] = value.into();
             }
             t += reso;
         }

src/api2/node/subscription.rs

@@ -1,13 +1,70 @@
-use anyhow::{Error};
-use serde_json::{json, Value};
+use anyhow::{Error, format_err, bail};
+use serde_json::Value;
 
 use proxmox::api::{api, Router, RpcEnvironment, Permission};
 
 use crate::tools;
-use crate::config::acl::PRIV_SYS_AUDIT;
+use crate::tools::subscription::{self, SubscriptionStatus, SubscriptionInfo};
+use crate::config::acl::{PRIV_SYS_AUDIT,PRIV_SYS_MODIFY};
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::api2::types::{NODE_SCHEMA, Userid};
 
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            force: {
+                description: "Always connect to server, even if information in cache is up to date.",
+                type: bool,
+                optional: true,
+                default: false,
+            },
+        },
+    },
+    protected: true,
+    access: {
+        permission: &Permission::Privilege(&["system"], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Check and update subscription status.
+fn check_subscription(
+    force: bool,
+) -> Result<(), Error> {
+    // FIXME: drop once proxmox-api-macro is bumped to >> 5.0.0-1
+    let _remove_me = API_METHOD_CHECK_SUBSCRIPTION_PARAM_DEFAULT_FORCE;
+
+    let info = match subscription::read_subscription() {
+        Err(err) => bail!("could not read subscription status: {}", err),
+        Ok(Some(info)) => info,
+        Ok(None) => return Ok(()),
+    };
+
+    let server_id = tools::get_hardware_address()?;
+    let key = if let Some(key) = info.key {
+        // always update apt auth if we have a key to ensure user can access enterprise repo
+        subscription::update_apt_auth(Some(key.to_owned()), Some(server_id.to_owned()))?;
+        key
+    } else {
+        String::new()
+    };
+
+    if !force && info.status == SubscriptionStatus::ACTIVE {
+        let age = proxmox::tools::time::epoch_i64() - info.checktime.unwrap_or(i64::MAX);
+        if age < subscription::MAX_LOCAL_KEY_AGE {
+            return Ok(());
+        }
+    }
+
+    let info = subscription::check_subscription(key, server_id)?;
+
+    subscription::write_subscription(info)
+        .map_err(|e| format_err!("Error writing updated subscription status - {}", e))?;
+
+    Ok(())
+}
+
 #[api(
     input: {
         properties: {
@@ -18,24 +75,7 @@ use crate::api2::types::{NODE_SCHEMA, Userid};
     },
     returns: {
         description: "Subscription status.",
-        properties: {
-            status: {
-                type: String,
-                description: "'NotFound', 'active' or 'inactive'."
-            },
-            message: {
-                type: String,
-                description: "Human readable problem description.",
-            },
-            serverid: {
-                type: String,
-                description: "The unique server ID, if permitted to access.",
-            },
-            url: {
-                type: String,
-                description: "URL to Web Shop.",
-            },
-        },
+        type: SubscriptionInfo,
     },
     access: {
         permission: &Permission::Anybody,
@@ -45,24 +85,95 @@ use crate::api2::types::{NODE_SCHEMA, Userid};
 fn get_subscription(
     _param: Value,
     rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
+) -> Result<SubscriptionInfo, Error> {
+    let url = "https://www.proxmox.com/en/proxmox-backup-server/pricing";
+
+    let info = match subscription::read_subscription() {
+        Err(err) => bail!("could not read subscription status: {}", err),
+        Ok(Some(info)) => info,
+        Ok(None) => SubscriptionInfo {
+            status: SubscriptionStatus::NOTFOUND,
+            message: Some("There is no subscription key".into()),
+            serverid: Some(tools::get_hardware_address()?),
+            url:  Some(url.into()),
+            ..Default::default()
+        },
+    };
+
     let userid: Userid = rpcenv.get_user().unwrap().parse()?;
     let user_info = CachedUserInfo::new()?;
     let user_privs = user_info.lookup_privs(&userid, &[]);
-    let server_id = if (user_privs & PRIV_SYS_AUDIT) != 0 {
-        tools::get_hardware_address()?
-    } else {
-        "hidden".to_string()
+
+    if (user_privs & PRIV_SYS_AUDIT) == 0 {
+        // not enough privileges for full state
+        return Ok(SubscriptionInfo {
+            status: info.status,
+            message: info.message,
+            url: info.url,
+            ..Default::default()
+        });
     };
 
-    let url = "https://www.proxmox.com/en/proxmox-backup-server/pricing";
-    Ok(json!({
-        "status": "NotFound",
-        "message": "There is no subscription key",
-        "serverid": server_id,
-        "url":  url,
-     }))
+    Ok(info)
+}
+
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            key: {
+                description: "Proxmox Backup Server subscription key",
+                type: String,
+                max_length: 32,
+            },
+        },
+    },
+    protected: true,
+    access: {
+        permission: &Permission::Privilege(&["system"], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Set a subscription key and check it.
+fn set_subscription(
+    key: String,
+) -> Result<(), Error> {
+
+    let server_id = tools::get_hardware_address()?;
+
+    let info = subscription::check_subscription(key, server_id.to_owned())?;
+
+    subscription::write_subscription(info)
+        .map_err(|e| format_err!("Error writing subscription status - {}", e))?;
+
+    Ok(())
+}
+
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+        },
+    },
+    protected: true,
+    access: {
+        permission: &Permission::Privilege(&["system"], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Delete subscription info.
+fn delete_subscription() -> Result<(), Error> {
+
+    subscription::delete_subscription()
+        .map_err(|err| format_err!("Deleting subscription failed: {}", err))?;
+
+    Ok(())
 }
 
 pub const ROUTER: Router = Router::new()
+    .post(&API_METHOD_CHECK_SUBSCRIPTION)
+    .put(&API_METHOD_SET_SUBSCRIPTION)
+    .delete(&API_METHOD_DELETE_SUBSCRIPTION)
     .get(&API_METHOD_GET_SUBSCRIPTION);

src/api2/node/tasks.rs

@@ -27,7 +27,7 @@ use crate::config::cached_user_info::CachedUserInfo;
         },
     },
     returns: {
-        description: "Task status nformation.",
+        description: "Task status information.",
         properties: {
             node: {
                 schema: NODE_SCHEMA,
@@ -72,7 +72,7 @@ use crate::config::cached_user_info::CachedUserInfo;
         },
     },
     access: {
-        description: "Users can access there own tasks, or need Sys.Audit on /system/tasks.",
+        description: "Users can access their own tasks, or need Sys.Audit on /system/tasks.",
         permission: &Permission::Anybody,
     },
 )]
@@ -342,7 +342,7 @@ pub fn list_tasks(
             if info.upid.worker_type == "backup" || info.upid.worker_type == "restore" ||
                 info.upid.worker_type == "prune"
             {
-                let prefix = format!("{}_", store);
+                let prefix = format!("{}:", store);
                 if !worker_id.starts_with(&prefix) { return None; }
             } else if info.upid.worker_type == "garbage_collection" {
                 if worker_id != store { return None; }

src/api2/ping.rs

@@ -0,0 +1,29 @@
+use anyhow::{Error};
+use serde_json::{json, Value};
+
+use proxmox::api::{api, Router, Permission};
+
+#[api(
+    returns: {
+        description: "Dummy ping",
+        type: Object,
+        properties: {
+            pong: {
+                description: "Always true",
+                type: bool,
+            }
+        }
+    },
+    access: {
+        description: "Anyone can access this, because it's used for a cheap check if the API daemon is online.",
+        permission: &Permission::World,
+    }
+)]
+/// Dummy method which replies with `{ "pong": True }`
+fn ping() -> Result<Value, Error> {
+    Ok(json!({
+        "pong": true,
+    }))
+}
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_PING);

src/api2/pull.rs

@@ -7,14 +7,13 @@ use futures::{select, future::FutureExt};
 use proxmox::api::api;
 use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
 
-use crate::server::{WorkerTask};
+use crate::server::{WorkerTask, jobstate::Job};
 use crate::backup::DataStore;
 use crate::client::{HttpClient, HttpClientOptions, BackupRepository, pull::pull_store};
 use crate::api2::types::*;
 use crate::config::{
     remote,
     sync::SyncJobConfig,
-    jobstate::Job,
     acl::{PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ},
     cached_user_info::CachedUserInfo,
 };

src/api2/reader.rs

@@ -14,9 +14,10 @@ use crate::api2::types::*;
 use crate::backup::*;
 use crate::server::{WorkerTask, H2Service};
 use crate::tools;
-use crate::config::acl::PRIV_DATASTORE_READ;
+use crate::config::acl::{PRIV_DATASTORE_READ, PRIV_DATASTORE_BACKUP};
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::api2::helpers;
+use crate::tools::fs::lock_dir_noblock_shared;
 
 mod environment;
 use environment::*;
@@ -58,7 +59,15 @@ fn upgrade_to_backup_reader_protocol(
         let store = tools::required_string_param(&param, "store")?.to_owned();
 
         let user_info = CachedUserInfo::new()?;
-        user_info.check_privs(&userid, &["datastore", &store], PRIV_DATASTORE_READ, false)?;
+        let privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+
+        let priv_read = privs & PRIV_DATASTORE_READ != 0;
+        let priv_backup = privs & PRIV_DATASTORE_BACKUP != 0;
+
+        // priv_backup needs owner check further down below!
+        if !priv_read && !priv_backup {
+            bail!("no permissions on /datastore/{}", store);
+        }
 
         let datastore = DataStore::lookup_datastore(&store)?;
 
@@ -83,11 +92,23 @@ fn upgrade_to_backup_reader_protocol(
         let env_type = rpcenv.env_type();
 
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
+        if !priv_read {
+            let owner = datastore.get_owner(backup_dir.group())?;
+            if owner != userid {
+                bail!("backup owner check failed!");
+            }
+        }
+
+        let _guard = lock_dir_noblock_shared(
+            &datastore.snapshot_path(&backup_dir),
+            "snapshot",
+            "locked by another operation")?;
+
         let path = datastore.base_path();
 
         //let files = BackupInfo::list_files(&path, &backup_dir)?;
 
-        let worker_id = format!("{}_{}_{}_{:08X}", store, backup_type, backup_id, backup_dir.backup_time());
+        let worker_id = format!("{}:{}/{}/{:08X}", store, backup_type, backup_id, backup_dir.backup_time());
 
         WorkerTask::spawn("reader", Some(worker_id), userid.clone(), true, move |worker| {
             let mut env = ReaderEnvironment::new(
@@ -131,11 +152,14 @@ fn upgrade_to_backup_reader_protocol(
 
             use futures::future::Either;
             futures::future::select(req_fut, abort_future)
-                .map(|res| match res {
-                    Either::Left((Ok(res), _)) => Ok(res),
-                    Either::Left((Err(err), _)) => Err(err),
-                    Either::Right((Ok(res), _)) => Ok(res),
-                    Either::Right((Err(err), _)) => Err(err),
+                .map(move |res| {
+                    let _guard = _guard;
+                    match res {
+                        Either::Left((Ok(res), _)) => Ok(res),
+                        Either::Left((Err(err), _)) => Err(err),
+                        Either::Right((Ok(res), _)) => Ok(res),
+                        Either::Right((Err(err), _)) => Err(err),
+                    }
                 })
                 .map_ok(move |_| env.log("reader finished successfully"))
         })?;
@@ -195,6 +219,27 @@ fn download_file(
 
         env.log(format!("download {:?}", path.clone()));
  
+        let index: Option<Box<dyn IndexFile + Send>> = match archive_type(&file_name)? {
+            ArchiveType::FixedIndex => {
+                let index = env.datastore.open_fixed_reader(&path)?;
+                Some(Box::new(index))
+            }
+            ArchiveType::DynamicIndex => {
+                let index = env.datastore.open_dynamic_reader(&path)?;
+                Some(Box::new(index))
+            }
+            _ => { None }
+        };
+
+        if let Some(index) = index {
+            env.log(format!("register chunks in '{}' as downloadable.", file_name));
+
+            for pos in 0..index.index_count() {
+                let info = index.chunk_info(pos).unwrap();
+                env.register_chunk(info.digest);
+            }
+        }
+
         helpers::create_download_response(path).await
     }.boxed()
 }
@@ -224,6 +269,11 @@ fn download_chunk(
         let digest_str = tools::required_string_param(&param, "digest")?;
         let digest = proxmox::tools::hex_to_digest(digest_str)?;
 
+        if !env.check_chunk_access(digest) {
+            env.log(format!("attempted to download chunk {} which is not in registered chunk list", digest_str));
+            return Err(http_err!(UNAUTHORIZED, "download chunk {} not allowed", digest_str));
+        }
+
         let (path, _) = env.datastore.chunk_path(&digest);
         let path2 = path.clone();
 

src/api2/reader/environment.rs

@@ -1,5 +1,5 @@
-//use anyhow::{bail, format_err, Error};
-use std::sync::Arc;
+use std::sync::{Arc,RwLock};
+use std::collections::HashSet;
 
 use serde_json::{json, Value};
 
@@ -23,7 +23,7 @@ pub struct ReaderEnvironment {
     pub worker: Arc<WorkerTask>,
     pub datastore: Arc<DataStore>,
     pub backup_dir: BackupDir,
-    // state: Arc<Mutex<SharedBackupState>>
+    allowed_chunks: Arc<RwLock<HashSet<[u8;32]>>>,
 }
 
 impl ReaderEnvironment {
@@ -45,7 +45,7 @@ impl ReaderEnvironment {
             debug: false,
             formatter: &JSON_FORMATTER,
             backup_dir,
-            //state: Arc::new(Mutex::new(state)),
+            allowed_chunks: Arc::new(RwLock::new(HashSet::new())),
         }
     }
 
@@ -57,6 +57,15 @@ impl ReaderEnvironment {
         if self.debug { self.worker.log(msg); }
     }
 
+
+    pub fn register_chunk(&self, digest: [u8;32]) {
+        let mut allowed_chunks = self.allowed_chunks.write().unwrap();
+        allowed_chunks.insert(digest);
+    }
+
+    pub fn check_chunk_access(&self, digest: [u8;32]) -> bool {
+       self.allowed_chunks.read().unwrap().contains(&digest)
+    }
 }
 
 impl RpcEnvironment for ReaderEnvironment {

src/api2/status.rs

@@ -17,6 +17,7 @@ use crate::api2::types::{
     RRDMode,
     RRDTimeFrameResolution,
     TaskListItem,
+    TaskStateType,
     Userid,
 };
 
@@ -186,6 +187,19 @@ fn datastore_status(
                 description: "Only list tasks since this UNIX epoch.",
                 optional: true,
             },
+            typefilter: {
+                optional: true,
+                type: String,
+                description: "Only list tasks, whose type contains this string.",
+            },
+            statusfilter: {
+                optional: true,
+                type: Array,
+                description: "Only list tasks which have any one of the listed status.",
+                items: {
+                    type: TaskStateType,
+                },
+            },
         },
     },
     returns: {
@@ -201,6 +215,8 @@ fn datastore_status(
 /// List tasks.
 pub fn list_tasks(
     since: Option<i64>,
+    typefilter: Option<String>,
+    statusfilter: Option<Vec<TaskStateType>>,
     _param: Value,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<TaskListItem>, Error> {
@@ -223,6 +239,27 @@ pub fn list_tasks(
             match info {
                 Ok(info) => {
                     if list_all || info.upid.userid == userid {
+                        if let Some(filter) = &typefilter {
+                            if !info.upid.worker_type.contains(filter) {
+                                return None;
+                            }
+                        }
+
+                        if let Some(filters) = &statusfilter {
+                            if let Some(state) = &info.state {
+                                let statetype = match state {
+                                    server::TaskState::OK { .. } => TaskStateType::OK,
+                                    server::TaskState::Unknown { .. } => TaskStateType::Unknown,
+                                    server::TaskState::Error { .. } => TaskStateType::Error,
+                                    server::TaskState::Warning { .. } => TaskStateType::Warning,
+                                };
+
+                                if !filters.contains(&statetype) {
+                                    return None;
+                                }
+                            }
+                        }
+
                         Some(Ok(TaskListItem::from(info)))
                     } else {
                         None

src/api2/types/mod.rs

@@ -302,7 +302,7 @@ pub const PRUNE_SCHEDULE_SCHEMA: Schema = StringSchema::new(
     .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
     .schema();
 
-pub const VERIFY_SCHEDULE_SCHEMA: Schema = StringSchema::new(
+pub const VERIFICATION_SCHEDULE_SCHEMA: Schema = StringSchema::new(
     "Run verify job at specified schedule.")
     .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
     .schema();
@@ -324,6 +324,16 @@ pub const REMOVE_VANISHED_BACKUPS_SCHEMA: Schema = BooleanSchema::new(
     .default(true)
     .schema();
 
+pub const IGNORE_VERIFIED_BACKUPS_SCHEMA: Schema = BooleanSchema::new(
+    "Do not verify backups that are already verified if their verification is not outdated.")
+    .default(true)
+    .schema();
+
+pub const VERIFICATION_OUTDATED_AFTER_SCHEMA: Schema = IntegerSchema::new(
+    "Days after that a verification becomes outdated")
+    .minimum(1)
+    .schema();
+
 pub const SINGLE_LINE_COMMENT_SCHEMA: Schema = StringSchema::new("Comment (single line).")
     .format(&SINGLE_LINE_COMMENT_FORMAT)
     .schema();
@@ -577,6 +587,8 @@ pub struct GarbageCollectionStatus {
     pub pending_chunks: usize,
     /// Number of chunks marked as .bad by verify that have been removed by GC.
     pub removed_bad: usize,
+    /// Number of chunks still marked as .bad after garbage collection.
+    pub still_bad: usize,
 }
 
 impl Default for GarbageCollectionStatus {
@@ -592,6 +604,7 @@ impl Default for GarbageCollectionStatus {
             pending_bytes: 0,
             pending_chunks: 0,
             removed_bad: 0,
+            still_bad: 0,
         }
     }
 }
@@ -662,6 +675,20 @@ impl From<crate::server::TaskListInfo> for TaskListItem {
     }
 }
 
+#[api()]
+#[derive(Eq, PartialEq, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum TaskStateType {
+    /// Ok
+    OK,
+    /// Warning
+    Warning,
+    /// Error
+    Error,
+    /// Unknown
+    Unknown,
+}
+
 #[api()]
 #[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "lowercase")]

src/api2/types/userid.rs

@@ -131,13 +131,13 @@ impl std::ops::Deref for Username {
 
 impl Borrow<UsernameRef> for Username {
     fn borrow(&self) -> &UsernameRef {
-        UsernameRef::new(self.as_str())
+        UsernameRef::new(self.0.as_str())
     }
 }
 
 impl AsRef<UsernameRef> for Username {
     fn as_ref(&self) -> &UsernameRef {
-        UsernameRef::new(self.as_str())
+        self.borrow()
     }
 }
 
@@ -204,13 +204,13 @@ impl std::ops::Deref for Realm {
 
 impl Borrow<RealmRef> for Realm {
     fn borrow(&self) -> &RealmRef {
-        RealmRef::new(self.as_str())
+        RealmRef::new(self.0.as_str())
     }
 }
 
 impl AsRef<RealmRef> for Realm {
     fn as_ref(&self) -> &RealmRef {
-        RealmRef::new(self.as_str())
+        self.borrow()
     }
 }
 
@@ -397,10 +397,7 @@ impl TryFrom<String> for Userid {
 
 impl PartialEq<str> for Userid {
     fn eq(&self, rhs: &str) -> bool {
-        rhs.len() > self.name_len + 2 // make sure range access below is allowed
-        && rhs.starts_with(self.name().as_str())
-        && rhs.as_bytes()[self.name_len] == b'@'
-        && &rhs[(self.name_len + 1)..] == self.realm().as_str()
+        self.data == *rhs
     }
 }
 

src/auth.rs

@@ -6,7 +6,6 @@ use std::process::{Command, Stdio};
 use std::io::Write;
 use std::ffi::{CString, CStr};
 
-use base64;
 use anyhow::{bail, format_err, Error};
 use serde_json::json;
 
@@ -25,8 +24,7 @@ impl ProxmoxAuthenticator for PAM {
         let mut auth = pam::Authenticator::with_password("proxmox-backup-auth").unwrap();
         auth.get_handler().set_credentials(username.as_str(), password);
         auth.authenticate()?;
-        return Ok(());
-
+        Ok(())
     }
 
     fn store_password(&self, username: &UsernameRef, password: &str) -> Result<(), Error> {
@@ -99,7 +97,7 @@ pub fn encrypt_pw(password: &str) -> Result<String, Error> {
 
 pub fn verify_crypt_pw(password: &str, enc_password: &str) -> Result<(), Error> {
     let verify = crypt(password.as_bytes(), enc_password)?;
-    if &verify != enc_password {
+    if verify != enc_password {
         bail!("invalid credentials");
     }
     Ok(())

src/backup.rs

@@ -1,107 +1,146 @@
-//! This module implements the proxmox backup data storage
+//! This module implements the data storage and access layer.
 //!
-//! Proxmox backup splits large files into chunks, and stores them
-//! deduplicated using a content addressable storage format.
+//! # Data formats
 //!
-//! A chunk is simply defined as binary blob, which is stored inside a
-//! `ChunkStore`, addressed by the SHA256 digest of the binary blob.
+//! PBS splits large files into chunks, and stores them deduplicated using
+//! a content addressable storage format.
 //!
-//! Index files are used to reconstruct the original file. They
-//! basically contain a list of SHA256 checksums. The `DynamicIndex*`
-//! format is able to deal with dynamic chunk sizes, whereas the
-//! `FixedIndex*` format is an optimization to store a list of equal
-//! sized chunks.
+//! Backup snapshots are stored as folders containing a manifest file and
+//! potentially one or more index or blob files.
 //!
-//! # ChunkStore Locking
+//! The manifest contains hashes of all other files and can be signed by
+//! the client.
 //!
-//! We need to be able to restart the proxmox-backup service daemons,
-//! so that we can update the software without rebooting the host. But
-//! such restarts must not abort running backup jobs, so we need to
-//! keep the old service running until those jobs are finished. This
-//! implies that we need some kind of locking for the
-//! ChunkStore. Please note that it is perfectly valid to have
-//! multiple parallel ChunkStore writers, even when they write the
-//! same chunk (because the chunk would have the same name and the
-//! same data). The only real problem is garbage collection, because
-//! we need to avoid deleting chunks which are still referenced.
+//! Blob files contain data directly. They are used for config files and
+//! the like.
 //!
-//! * Read Index Files:
+//! Index files are used to reconstruct an original file. They contain a
+//! list of SHA256 checksums. The `DynamicIndex*` format is able to deal
+//! with dynamic chunk sizes (CT and host backups), whereas the
+//! `FixedIndex*` format is an optimization to store a list of equal sized
+//! chunks (VMs, whole block devices).
 //!
-//!   Acquire shared lock for .idx files.
-//!
-//!
-//! * Delete Index Files:
-//!
-//!   Acquire exclusive lock for .idx files. This makes sure that we do
-//!   not delete index files while they are still in use.
-//!
-//!
-//! * Create Index Files:
-//!
-//!   Acquire shared lock for ChunkStore (process wide).
-//!
-//!   Note: When creating .idx files, we create temporary a (.tmp) file,
-//!   then do an atomic rename ...
-//!
-//!
-//! * Garbage Collect:
-//!
-//!   Acquire exclusive lock for ChunkStore (process wide). If we have
-//!   already a shared lock for the ChunkStore, try to upgrade that
-//!   lock.
-//!
-//!
-//! * Server Restart
-//!
-//!   Try to abort the running garbage collection to release exclusive
-//!   ChunkStore locks ASAP. Start the new service with the existing listening
-//!   socket.
+//! A chunk is defined as a binary blob, which is stored inside a
+//! [ChunkStore](struct.ChunkStore.html) instead of the backup directory
+//! directly, and can be addressed by its SHA256 digest.
 //!
 //!
 //! # Garbage Collection (GC)
 //!
-//! Deleting backups is as easy as deleting the corresponding .idx
-//! files. Unfortunately, this does not free up any storage, because
-//! those files just contain references to chunks.
+//! Deleting backups is as easy as deleting the corresponding .idx files.
+//! However, this does not free up any storage, because those files just
+//! contain references to chunks.
 //!
 //! To free up some storage, we run a garbage collection process at
-//! regular intervals. The collector uses a mark and sweep
-//! approach. In the first phase, it scans all .idx files to mark used
-//! chunks. The second phase then removes all unmarked chunks from the
-//! store.
+//! regular intervals. The collector uses a mark and sweep approach. In
+//! the first phase, it scans all .idx files to mark used chunks. The
+//! second phase then removes all unmarked chunks from the store.
 //!
-//! The above locking mechanism makes sure that we are the only
-//! process running GC. But we still want to be able to create backups
-//! during GC, so there may be multiple backup threads/tasks
-//! running. Either started before GC started, or started while GC is
-//! running.
+//! The locking mechanisms mentioned below make sure that we are the only
+//! process running GC. We still want to be able to create backups during
+//! GC, so there may be multiple backup threads/tasks running, either
+//! started before GC, or while GC is running.
 //!
 //! ## `atime` based GC
 //!
 //! The idea here is to mark chunks by updating the `atime` (access
-//! timestamp) on the chunk file. This is quite simple and does not
-//! need additional RAM.
+//! timestamp) on the chunk file. This is quite simple and does not need
+//! additional RAM.
 //!
 //! One minor problem is that recent Linux versions use the `relatime`
-//! mount flag by default for performance reasons (yes, we want
-//! that). When enabled, `atime` data is written to the disk only if
-//! the file has been modified since the `atime` data was last updated
-//! (`mtime`), or if the file was last accessed more than a certain
-//! amount of time ago (by default 24h). So we may only delete chunks
-//! with `atime` older than 24 hours.
-//!
-//! Another problem arises from running backups. The mark phase does
-//! not find any chunks from those backups, because there is no .idx
-//! file for them (created after the backup). Chunks created or
-//! touched by those backups may have an `atime` as old as the start
-//! time of those backups. Please note that the backup start time may
-//! predate the GC start time. So we may only delete chunks older than
-//! the start time of those running backup jobs.
+//! mount flag by default for performance reasons (and we want that). When
+//! enabled, `atime` data is written to the disk only if the file has been
+//! modified since the `atime` data was last updated (`mtime`), or if the
+//! file was last accessed more than a certain amount of time ago (by
+//! default 24h). So we may only delete chunks with `atime` older than 24
+//! hours.
 //!
+//! Another problem arises from running backups. The mark phase does not
+//! find any chunks from those backups, because there is no .idx file for
+//! them (created after the backup). Chunks created or touched by those
+//! backups may have an `atime` as old as the start time of those backups.
+//! Please note that the backup start time may predate the GC start time.
+//! So we may only delete chunks older than the start time of those
+//! running backup jobs, which might be more than 24h back (this is the
+//! reason why ProcessLocker exclusive locks only have to be exclusive
+//! between processes, since within one we can determine the age of the
+//! oldest shared lock).
 //!
 //! ## Store `marks` in RAM using a HASH
 //!
-//! Not sure if this is better. TODO
+//! Might be better. Under investigation.
+//!
+//!
+//! # Locking
+//!
+//! Since PBS allows multiple potentially interfering operations at the
+//! same time (e.g. garbage collect, prune, multiple backup creations
+//! (only in seperate groups), forget, ...), these need to lock against
+//! each other in certain scenarios. There is no overarching global lock
+//! though, instead always the finest grained lock possible is used,
+//! because running these operations concurrently is treated as a feature
+//! on its own.
+//!
+//! ## Inter-process Locking
+//!
+//! We need to be able to restart the proxmox-backup service daemons, so
+//! that we can update the software without rebooting the host. But such
+//! restarts must not abort running backup jobs, so we need to keep the
+//! old service running until those jobs are finished. This implies that
+//! we need some kind of locking for modifying chunks and indices in the
+//! ChunkStore.
+//!
+//! Please note that it is perfectly valid to have multiple
+//! parallel ChunkStore writers, even when they write the same chunk
+//! (because the chunk would have the same name and the same data, and
+//! writes are completed atomically via a rename). The only problem is
+//! garbage collection, because we need to avoid deleting chunks which are
+//! still referenced.
+//!
+//! To do this we use the
+//! [ProcessLocker](../tools/struct.ProcessLocker.html).
+//!
+//! ### ChunkStore-wide
+//!
+//! * Create Index Files:
+//!
+//!   Acquire shared lock for ChunkStore.
+//!
+//!   Note: When creating .idx files, we create a temporary .tmp file,
+//!   then do an atomic rename.
+//!
+//! * Garbage Collect:
+//!
+//!   Acquire exclusive lock for ChunkStore. If we have
+//!   already a shared lock for the ChunkStore, try to upgrade that
+//!   lock.
+//!
+//! Exclusive locks only work _between processes_. It is valid to have an
+//! exclusive and one or more shared locks held within one process. Writing
+//! chunks within one process is synchronized using the gc_mutex.
+//!
+//! On server restart, we stop any running GC in the old process to avoid
+//! having the exclusive lock held for too long.
+//!
+//! ## Locking table
+//!
+//! Below table shows all operations that play a role in locking, and which
+//! mechanisms are used to make their concurrent usage safe.
+//!
+//! | starting ><br>v during | read index file | create index file | GC mark | GC sweep | update manifest | forget | prune | create backup | verify | reader api |
+//! |-|-|-|-|-|-|-|-|-|-|-|
+//! | **read index file** | / | / | / | / | / | mmap stays valid, oldest_shared_lock prevents GC | see forget column | / | / | / |
+//! | **create index file** | / | / | / | / | / | / | / | /, happens at the end, after all chunks are touched | /, only happens without a manifest | / |
+//! | **GC mark** | / | Datastore process-lock shared | gc_mutex, exclusive ProcessLocker | gc_mutex | /, GC only cares about index files, not manifests | tells GC about removed chunks | see forget column | /, index files don’t exist yet | / | / |
+//! | **GC sweep** | / | Datastore process-lock shared | gc_mutex, exclusive ProcessLocker | gc_mutex | / | /, chunks already marked | see forget column | chunks get touched; chunk_store.mutex; oldest PL lock | / | / |
+//! | **update manifest** | / | / | / | / | update_manifest lock | update_manifest lock, remove dir under lock | see forget column | /, “write manifest” happens at the end | /, can call “write manifest”, see that column | / |
+//! | **forget** | / | / | removed_during_gc mutex is held during unlink | marking done, doesn’t matter if forgotten now | update_manifest lock, forget waits for lock | /, unlink is atomic | causes forget to fail, but that’s OK | running backup has snapshot flock | /, potentially detects missing folder | shared snap flock |
+//! | **prune** | / | / | see forget row | see forget row | see forget row | causes warn in prune, but no error | see forget column | running and last non-running can’t be pruned | see forget row | shared snap flock |
+//! | **create backup** | / | only time this happens, thus has snapshot flock | / | chunks get touched; chunk_store.mutex; oldest PL lock | no lock, but cannot exist beforehand | snapshot flock, can’t be forgotten | running and last non-running can’t be pruned | snapshot group flock, only one running per group | /, won’t be verified since manifest missing | / |
+//! | **verify** | / | / | / | / | see “update manifest” row | /, potentially detects missing folder | see forget column | / | /, but useless (“update manifest” protects itself) | / |
+//! | **reader api** | / | / | / | /, open snap can’t be forgotten, so ref must exist | / | prevented by shared snap flock | prevented by shared snap flock | / | / | /, lock is shared |!
+//! * / = no interaction
+//! * shared/exclusive from POV of 'starting' process
 
 use anyhow::{bail, Error};
 

src/backup/async_index_reader.rs

@@ -15,6 +15,17 @@ use super::IndexFile;
 use super::read_chunk::AsyncReadChunk;
 use super::index::ChunkReadInfo;
 
+// FIXME: This enum may not be required?
+// - Put the `WaitForData` case directly into a `read_future: Option<>`
+// - make the read loop as follows:
+//   * if read_buffer is not empty:
+//        use it
+//   * else if read_future is there:
+//        poll it
+//        if read: move data to read_buffer
+//   * else
+//        create read future
+#[allow(clippy::enum_variant_names)]
 enum AsyncIndexReaderState<S> {
     NoData,
     WaitForData(Pin<Box<dyn Future<Output = Result<(S, Vec<u8>), Error>> + Send + 'static>>),
@@ -118,9 +129,8 @@ where
                 }
                 AsyncIndexReaderState::WaitForData(ref mut future) => {
                     match ready!(future.as_mut().poll(cx)) {
-                        Ok((store, mut chunk_data)) => {
-                            this.read_buffer.clear();
-                            this.read_buffer.append(&mut chunk_data);
+                        Ok((store, chunk_data)) => {
+                            this.read_buffer = chunk_data;
                             this.state = AsyncIndexReaderState::HaveData;
                             this.store = Some(store);
                         }

src/backup/chunk_store.rs

@@ -11,7 +11,7 @@ use crate::tools;
 use crate::api2::types::GarbageCollectionStatus;
 
 use super::DataBlob;
-use crate::server::WorkerTask;
+use crate::task::TaskState;
 
 /// File system based chunk store
 pub struct ChunkStore {
@@ -278,7 +278,7 @@ impl ChunkStore {
         oldest_writer: i64,
         phase1_start_time: i64,
         status: &mut GarbageCollectionStatus,
-        worker: &WorkerTask,
+        worker: &dyn TaskState,
     ) -> Result<(), Error> {
         use nix::sys::stat::fstatat;
         use nix::unistd::{unlinkat, UnlinkatFlags};
@@ -297,10 +297,15 @@ impl ChunkStore {
         for (entry, percentage, bad) in self.get_chunk_iterator()? {
             if last_percentage != percentage {
                 last_percentage = percentage;
-                worker.log(format!("percentage done: phase2 {}% (processed {} chunks)", percentage, chunk_count));
+                crate::task_log!(
+                    worker,
+                    "percentage done: phase2 {}% (processed {} chunks)",
+                    percentage,
+                    chunk_count,
+                );
             }
 
-            worker.fail_on_abort()?;
+            worker.check_abort()?;
             tools::fail_on_shutdown()?;
 
             let (dirfd, entry) = match entry {
@@ -334,12 +339,13 @@ impl ChunkStore {
                         Ok(_) => {
                             match unlinkat(Some(dirfd), filename, UnlinkatFlags::NoRemoveDir) {
                                 Err(err) =>
-                                    worker.warn(format!(
+                                    crate::task_warn!(
+                                        worker,
                                         "unlinking corrupt chunk {:?} failed on store '{}' - {}",
                                         filename,
                                         self.name,
                                         err,
-                                    )),
+                                    ),
                                 Ok(_) => {
                                     status.removed_bad += 1;
                                     status.removed_bytes += stat.st_size as u64;
@@ -348,14 +354,17 @@ impl ChunkStore {
                         },
                         Err(nix::Error::Sys(nix::errno::Errno::ENOENT)) => {
                             // chunk hasn't been rewritten yet, keep .bad file
+                            status.still_bad += 1;
                         },
                         Err(err) => {
                             // some other error, warn user and keep .bad file around too
-                            worker.warn(format!(
+                            status.still_bad += 1;
+                            crate::task_warn!(
+                                worker,
                                 "error during stat on '{:?}' - {}",
                                 orig_filename,
                                 err,
-                            ));
+                            );
                         }
                     }
                 } else if stat.st_atime < min_atime {
@@ -371,14 +380,12 @@ impl ChunkStore {
                     }
                     status.removed_chunks += 1;
                     status.removed_bytes += stat.st_size as u64;
+                } else if stat.st_atime < oldest_writer {
+                    status.pending_chunks += 1;
+                    status.pending_bytes += stat.st_size as u64;
                 } else {
-                    if stat.st_atime < oldest_writer {
-                        status.pending_chunks += 1;
-                        status.pending_bytes += stat.st_size as u64;
-                    } else {
-                        status.disk_chunks += 1;
-                        status.disk_bytes += stat.st_size as u64;
-                     }
+                    status.disk_chunks += 1;
+                    status.disk_bytes += stat.st_size as u64;
                 }
             }
             drop(lock);

src/backup/datastore.rs

@@ -3,26 +3,28 @@ use std::io::{self, Write};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 use std::convert::TryFrom;
+use std::time::Duration;
+use std::fs::File;
 
 use anyhow::{bail, format_err, Error};
 use lazy_static::lazy_static;
-use serde_json::Value;
 
-use proxmox::tools::fs::{replace_file, CreateOptions};
+use proxmox::tools::fs::{replace_file, file_read_optional_string, CreateOptions, open_file_locked};
 
 use super::backup_info::{BackupGroup, BackupDir};
 use super::chunk_store::ChunkStore;
 use super::dynamic_index::{DynamicIndexReader, DynamicIndexWriter};
 use super::fixed_index::{FixedIndexReader, FixedIndexWriter};
-use super::manifest::{MANIFEST_BLOB_NAME, CLIENT_LOG_BLOB_NAME, BackupManifest};
+use super::manifest::{MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME, CLIENT_LOG_BLOB_NAME, BackupManifest};
 use super::index::*;
 use super::{DataBlob, ArchiveType, archive_type};
-use crate::config::datastore;
-use crate::server::WorkerTask;
+use crate::config::datastore::{self, DataStoreConfig};
+use crate::task::TaskState;
 use crate::tools;
 use crate::tools::format::HumanByte;
 use crate::tools::fs::{lock_dir_noblock, DirLockGuard};
 use crate::api2::types::{GarbageCollectionStatus, Userid};
+use crate::server::UPID;
 
 lazy_static! {
     static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new());
@@ -36,6 +38,7 @@ pub struct DataStore {
     chunk_store: Arc<ChunkStore>,
     gc_mutex: Mutex<bool>,
     last_gc_status: Mutex<GarbageCollectionStatus>,
+    verify_new: bool,
 }
 
 impl DataStore {
@@ -44,17 +47,20 @@ impl DataStore {
 
         let (config, _digest) = datastore::config()?;
         let config: datastore::DataStoreConfig = config.lookup("datastore", name)?;
+        let path = PathBuf::from(&config.path);
 
         let mut map = DATASTORE_MAP.lock().unwrap();
 
         if let Some(datastore) = map.get(name) {
             // Compare Config - if changed, create new Datastore object!
-            if datastore.chunk_store.base == PathBuf::from(&config.path) {
+            if datastore.chunk_store.base == path &&
+                datastore.verify_new == config.verify_new.unwrap_or(false)
+            {
                 return Ok(datastore.clone());
             }
         }
 
-        let datastore = DataStore::open(name)?;
+        let datastore = DataStore::open_with_path(name, &path, config)?;
 
         let datastore = Arc::new(datastore);
         map.insert(name.to_string(), datastore.clone());
@@ -62,26 +68,29 @@ impl DataStore {
         Ok(datastore)
     }
 
-    pub fn open(store_name: &str) -> Result<Self, Error> {
-
-        let (config, _digest) = datastore::config()?;
-        let (_, store_config) = config.sections.get(store_name)
-            .ok_or(format_err!("no such datastore '{}'", store_name))?;
-
-        let path = store_config["path"].as_str().unwrap();
-
-        Self::open_with_path(store_name, Path::new(path))
-    }
-
-    pub fn open_with_path(store_name: &str, path: &Path) -> Result<Self, Error> {
+    fn open_with_path(store_name: &str, path: &Path, config: DataStoreConfig) -> Result<Self, Error> {
         let chunk_store = ChunkStore::open(store_name, path)?;
 
-        let gc_status = GarbageCollectionStatus::default();
+        let mut gc_status_path = chunk_store.base_path();
+        gc_status_path.push(".gc-status");
+
+        let gc_status = if let Some(state) = file_read_optional_string(gc_status_path)? {
+            match serde_json::from_str(&state) {
+                Ok(state) => state,
+                Err(err) => {
+                    eprintln!("error reading gc-status: {}", err);
+                    GarbageCollectionStatus::default()
+                }
+            }
+        } else {
+            GarbageCollectionStatus::default()
+        };
 
         Ok(Self {
             chunk_store: Arc::new(chunk_store),
             gc_mutex: Mutex::new(false),
             last_gc_status: Mutex::new(gc_status),
+            verify_new: config.verify_new.unwrap_or(false),
         })
     }
 
@@ -207,10 +216,17 @@ impl DataStore {
         let _guard = tools::fs::lock_dir_noblock(&full_path, "backup group", "possible running backup")?;
 
         log::info!("removing backup group {:?}", full_path);
+
+        // remove all individual backup dirs first to ensure nothing is using them
+        for snap in backup_group.list_backups(&self.base_path())? {
+            self.remove_backup_dir(&snap.backup_dir, false)?;
+        }
+
+        // no snapshots left, we can now safely remove the empty folder
         std::fs::remove_dir_all(&full_path)
             .map_err(|err| {
                 format_err!(
-                    "removing backup group {:?} failed - {}",
+                    "removing backup group directory {:?} failed - {}",
                     full_path,
                     err,
                 )
@@ -224,9 +240,10 @@ impl DataStore {
 
         let full_path = self.snapshot_path(backup_dir);
 
-        let _guard;
+        let (_guard, _manifest_guard);
         if !force {
-            _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or used as base")?;
+            _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or in use")?;
+            _manifest_guard = self.lock_manifest(backup_dir);
         }
 
         log::info!("removing backup snapshot {:?}", full_path);
@@ -292,7 +309,7 @@ impl DataStore {
         let mut file = open_options.open(&path)
             .map_err(|err| format_err!("unable to create owner file {:?} - {}", path, err))?;
 
-        write!(file, "{}\n", userid)
+        writeln!(file, "{}", userid)
             .map_err(|err| format_err!("unable to write owner file  {:?} - {}", path, err))?;
 
         Ok(())
@@ -411,25 +428,34 @@ impl DataStore {
         index: I,
         file_name: &Path, // only used for error reporting
         status: &mut GarbageCollectionStatus,
-        worker: &WorkerTask,
+        worker: &dyn TaskState,
     ) -> Result<(), Error> {
 
         status.index_file_count += 1;
         status.index_data_bytes += index.index_bytes();
 
         for pos in 0..index.index_count() {
-            worker.fail_on_abort()?;
+            worker.check_abort()?;
             tools::fail_on_shutdown()?;
             let digest = index.index_digest(pos).unwrap();
             if let Err(err) = self.chunk_store.touch_chunk(digest) {
-                worker.warn(&format!("warning: unable to access chunk {}, required by {:?} - {}",
-                      proxmox::tools::digest_to_hex(digest), file_name, err));
+                crate::task_warn!(
+                    worker,
+                    "warning: unable to access chunk {}, required by {:?} - {}",
+                    proxmox::tools::digest_to_hex(digest),
+                    file_name,
+                    err,
+                );
             }
         }
         Ok(())
     }
 
-    fn mark_used_chunks(&self, status: &mut GarbageCollectionStatus, worker: &WorkerTask) -> Result<(), Error> {
+    fn mark_used_chunks(
+        &self,
+        status: &mut GarbageCollectionStatus,
+        worker: &dyn TaskState,
+    ) -> Result<(), Error> {
 
         let image_list = self.list_images()?;
 
@@ -441,24 +467,41 @@ impl DataStore {
 
         for path in image_list {
 
-            worker.fail_on_abort()?;
+            worker.check_abort()?;
             tools::fail_on_shutdown()?;
 
-            if let Ok(archive_type) = archive_type(&path) {
-                if archive_type == ArchiveType::FixedIndex {
-                    let index = self.open_fixed_reader(&path)?;
-                    self.index_mark_used_chunks(index, &path, status, worker)?;
-                } else if archive_type == ArchiveType::DynamicIndex {
-                    let index = self.open_dynamic_reader(&path)?;
-                    self.index_mark_used_chunks(index, &path, status, worker)?;
+            let full_path = self.chunk_store.relative_path(&path);
+            match std::fs::File::open(&full_path) {
+                Ok(file) => {
+                    if let Ok(archive_type) = archive_type(&path) {
+                        if archive_type == ArchiveType::FixedIndex {
+                            let index = FixedIndexReader::new(file)?;
+                            self.index_mark_used_chunks(index, &path, status, worker)?;
+                        } else if archive_type == ArchiveType::DynamicIndex {
+                            let index = DynamicIndexReader::new(file)?;
+                            self.index_mark_used_chunks(index, &path, status, worker)?;
+                        }
+                    }
+                }
+                Err(err) => {
+                    if err.kind() == std::io::ErrorKind::NotFound {
+                        // simply ignore vanished files
+                    } else {
+                        return Err(err.into());
+                    }
                 }
             }
             done += 1;
 
             let percentage = done*100/image_count;
             if percentage > last_percentage {
-                worker.log(format!("percentage done: phase1 {}% ({} of {} index files)",
-                                   percentage, done, image_count));
+                crate::task_log!(
+                    worker,
+                    "percentage done: phase1 {}% ({} of {} index files)",
+                    percentage,
+                    done,
+                    image_count,
+                );
                 last_percentage = percentage;
             }
         }
@@ -474,7 +517,7 @@ impl DataStore {
         if let Ok(_) = self.gc_mutex.try_lock() { false } else { true }
     }
 
-    pub fn garbage_collection(&self, worker: &WorkerTask) -> Result<(), Error> {
+    pub fn garbage_collection(&self, worker: &dyn TaskState, upid: &UPID) -> Result<(), Error> {
 
         if let Ok(ref mut _mutex) = self.gc_mutex.try_lock() {
 
@@ -487,36 +530,76 @@ impl DataStore {
             let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(phase1_start_time);
 
             let mut gc_status = GarbageCollectionStatus::default();
-            gc_status.upid = Some(worker.to_string());
+            gc_status.upid = Some(upid.to_string());
 
-            worker.log("Start GC phase1 (mark used chunks)");
+            crate::task_log!(worker, "Start GC phase1 (mark used chunks)");
 
-            self.mark_used_chunks(&mut gc_status, &worker)?;
+            self.mark_used_chunks(&mut gc_status, worker)?;
 
-            worker.log("Start GC phase2 (sweep unused chunks)");
-            self.chunk_store.sweep_unused_chunks(oldest_writer, phase1_start_time, &mut gc_status, &worker)?;
+            crate::task_log!(worker, "Start GC phase2 (sweep unused chunks)");
+            self.chunk_store.sweep_unused_chunks(
+                oldest_writer,
+                phase1_start_time,
+                &mut gc_status,
+                worker,
+            )?;
 
-            worker.log(&format!("Removed garbage: {}", HumanByte::from(gc_status.removed_bytes)));
-            worker.log(&format!("Removed chunks: {}", gc_status.removed_chunks));
+            crate::task_log!(
+                worker,
+                "Removed garbage: {}",
+                HumanByte::from(gc_status.removed_bytes),
+            );
+            crate::task_log!(worker, "Removed chunks: {}", gc_status.removed_chunks);
             if gc_status.pending_bytes > 0 {
-                worker.log(&format!("Pending removals: {} (in {} chunks)", HumanByte::from(gc_status.pending_bytes), gc_status.pending_chunks));
+                crate::task_log!(
+                    worker,
+                    "Pending removals: {} (in {} chunks)",
+                    HumanByte::from(gc_status.pending_bytes),
+                    gc_status.pending_chunks,
+                );
             }
             if gc_status.removed_bad > 0 {
-                worker.log(&format!("Removed bad files: {}", gc_status.removed_bad));
+                crate::task_log!(worker, "Removed bad files: {}", gc_status.removed_bad);
             }
 
-            worker.log(&format!("Original data usage: {}", HumanByte::from(gc_status.index_data_bytes)));
+            crate::task_log!(
+                worker,
+                "Original data usage: {}",
+                HumanByte::from(gc_status.index_data_bytes),
+            );
 
             if gc_status.index_data_bytes > 0 {
                 let comp_per = (gc_status.disk_bytes as f64 * 100.)/gc_status.index_data_bytes as f64;
-                worker.log(&format!("On-Disk usage: {} ({:.2}%)", HumanByte::from(gc_status.disk_bytes), comp_per));
+                crate::task_log!(
+                    worker,
+                    "On-Disk usage: {} ({:.2}%)",
+                    HumanByte::from(gc_status.disk_bytes),
+                    comp_per,
+                );
             }
 
-            worker.log(&format!("On-Disk chunks: {}", gc_status.disk_chunks));
+            crate::task_log!(worker, "On-Disk chunks: {}", gc_status.disk_chunks);
 
             if gc_status.disk_chunks > 0 {
                 let avg_chunk = gc_status.disk_bytes/(gc_status.disk_chunks as u64);
-                worker.log(&format!("Average chunk size: {}", HumanByte::from(avg_chunk)));
+                crate::task_log!(worker, "Average chunk size: {}", HumanByte::from(avg_chunk));
+            }
+
+            if let Ok(serialized) = serde_json::to_string(&gc_status) {
+                let mut path = self.base_path();
+                path.push(".gc-status");
+
+                let backup_user = crate::backup::backup_user()?;
+                let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644);
+                // set the correct owner/group/permissions while saving file
+                // owner(rw) = backup, group(r)= backup
+                let options = CreateOptions::new()
+                    .perm(mode)
+                    .owner(backup_user.uid)
+                    .group(backup_user.gid);
+
+                // ignore errors
+                let _ = replace_file(path, serialized.as_bytes(), options);
             }
 
             *self.last_gc_status.lock().unwrap() = gc_status;
@@ -573,8 +656,27 @@ impl DataStore {
             digest_str,
             err,
         ))
-     }
+    }
 
+    fn lock_manifest(
+        &self,
+        backup_dir: &BackupDir,
+    ) -> Result<File, Error> {
+        let mut path = self.base_path();
+        path.push(backup_dir.relative_path());
+        path.push(&MANIFEST_LOCK_NAME);
+
+        // update_manifest should never take a long time, so if someone else has
+        // the lock we can simply block a bit and should get it soon
+        open_file_locked(&path, Duration::from_secs(5), true)
+            .map_err(|err| {
+                format_err!(
+                    "unable to acquire manifest lock {:?} - {}", &path, err
+                )
+            })
+    }
+
+    /// Load the manifest without a lock. Must not be written back.
     pub fn load_manifest(
         &self,
         backup_dir: &BackupDir,
@@ -585,22 +687,20 @@ impl DataStore {
         Ok((manifest, raw_size))
     }
 
-    pub fn load_manifest_json(
+    /// Update the manifest of the specified snapshot. Never write a manifest directly,
+    /// only use this method - anything else may break locking guarantees.
+    pub fn update_manifest(
         &self,
         backup_dir: &BackupDir,
-    ) -> Result<Value, Error> {
-        let blob = self.load_blob(backup_dir, MANIFEST_BLOB_NAME)?;
-        // no expected digest available
-        let manifest_data = blob.decode(None, None)?;
-        let manifest: Value = serde_json::from_slice(&manifest_data[..])?;
-        Ok(manifest)
-    }
-
-    pub fn store_manifest(
-        &self,
-        backup_dir: &BackupDir,
-        manifest: Value,
+        update_fn: impl FnOnce(&mut BackupManifest),
     ) -> Result<(), Error> {
+
+        let _guard = self.lock_manifest(backup_dir)?;
+        let (mut manifest, _) = self.load_manifest(&backup_dir)?;
+
+        update_fn(&mut manifest);
+
+        let manifest = serde_json::to_value(manifest)?;
         let manifest = serde_json::to_string_pretty(&manifest)?;
         let blob = DataBlob::encode(manifest.as_bytes(), None, true)?;
         let raw_data = blob.raw_data();
@@ -609,8 +709,13 @@ impl DataStore {
         path.push(backup_dir.relative_path());
         path.push(MANIFEST_BLOB_NAME);
 
+        // atomic replace invalidates flock - no other writes past this point!
         replace_file(&path, raw_data, CreateOptions::new())?;
 
         Ok(())
     }
+
+    pub fn verify_new(&self) -> bool {
+        self.verify_new
+    }
 }

src/backup/dynamic_index.rs

@@ -90,12 +90,6 @@ impl DynamicIndexReader {
     }
 
     pub fn new(mut file: std::fs::File) -> Result<Self, Error> {
-        if let Err(err) =
-            nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock)
-        {
-            bail!("unable to get shared lock - {}", err);
-        }
-
         // FIXME: This is NOT OUR job! Check the callers of this method and remove this!
         file.seek(SeekFrom::Start(0))?;
 

src/backup/fixed_index.rs

@@ -65,12 +65,6 @@ impl FixedIndexReader {
     }
 
     pub fn new(mut file: std::fs::File) -> Result<Self, Error> {
-        if let Err(err) =
-            nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock)
-        {
-            bail!("unable to get shared lock - {}", err);
-        }
-
         file.seek(SeekFrom::Start(0))?;
 
         let header_size = std::mem::size_of::<FixedIndexHeader>();

src/backup/manifest.rs

@@ -8,6 +8,7 @@ use ::serde::{Deserialize, Serialize};
 use crate::backup::{BackupDir, CryptMode, CryptConfig};
 
 pub const MANIFEST_BLOB_NAME: &str = "index.json.blob";
+pub const MANIFEST_LOCK_NAME: &str = ".index.json.lck";
 pub const CLIENT_LOG_BLOB_NAME: &str = "client.log.blob";
 
 mod hex_csum {

src/backup/verify.rs

@@ -2,13 +2,12 @@ use std::collections::HashSet;
 use std::sync::{Arc, Mutex};
 use std::sync::atomic::{Ordering, AtomicUsize};
 use std::time::Instant;
+use nix::dir::Dir;
 
 use anyhow::{bail, format_err, Error};
 
 use crate::{
-    server::WorkerTask,
     api2::types::*,
-    tools::ParallelHandler,
     backup::{
         DataStore,
         DataBlob,
@@ -21,6 +20,11 @@ use crate::{
         ArchiveType,
         archive_type,
     },
+    server::UPID,
+    task::TaskState,
+    task_log,
+    tools::ParallelHandler,
+    tools::fs::lock_dir_noblock_shared,
 };
 
 fn verify_blob(datastore: Arc<DataStore>, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
@@ -51,7 +55,7 @@ fn verify_blob(datastore: Arc<DataStore>, backup_dir: &BackupDir, info: &FileInf
 fn rename_corrupted_chunk(
     datastore: Arc<DataStore>,
     digest: &[u8;32],
-    worker: Arc<WorkerTask>,
+    worker: &dyn TaskState,
 ) {
     let (path, digest_str) = datastore.chunk_path(digest);
 
@@ -64,12 +68,12 @@ fn rename_corrupted_chunk(
 
     match std::fs::rename(&path, &new_path) {
         Ok(_) => {
-            worker.log(format!("corrupted chunk renamed to {:?}", &new_path));
+            task_log!(worker, "corrupted chunk renamed to {:?}", &new_path);
         },
         Err(err) => {
             match err.kind() {
                 std::io::ErrorKind::NotFound => { /* ignored */ },
-                _ => worker.log(format!("could not rename corrupted chunk {:?} - {}", &path, err))
+                _ => task_log!(worker, "could not rename corrupted chunk {:?} - {}", &path, err)
             }
         }
     };
@@ -81,7 +85,7 @@ fn verify_index_chunks(
     verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
     corrupt_chunks: Arc<Mutex<HashSet<[u8; 32]>>>,
     crypt_mode: CryptMode,
-    worker: Arc<WorkerTask>,
+    worker: Arc<dyn TaskState + Send + Sync>,
 ) -> Result<(), Error> {
 
     let errors = Arc::new(AtomicUsize::new(0));
@@ -103,7 +107,7 @@ fn verify_index_chunks(
             let chunk_crypt_mode = match chunk.crypt_mode() {
                 Err(err) => {
                     corrupt_chunks2.lock().unwrap().insert(digest);
-                    worker2.log(format!("can't verify chunk, unknown CryptMode - {}", err));
+                    task_log!(worker2, "can't verify chunk, unknown CryptMode - {}", err);
                     errors2.fetch_add(1, Ordering::SeqCst);
                     return Ok(());
                 },
@@ -111,19 +115,20 @@ fn verify_index_chunks(
             };
 
             if chunk_crypt_mode != crypt_mode {
-                worker2.log(format!(
+                task_log!(
+                    worker2,
                     "chunk CryptMode {:?} does not match index CryptMode {:?}",
                     chunk_crypt_mode,
                     crypt_mode
-                ));
+                );
                 errors2.fetch_add(1, Ordering::SeqCst);
             }
 
             if let Err(err) = chunk.verify_unencrypted(size as usize, &digest) {
                 corrupt_chunks2.lock().unwrap().insert(digest);
-                worker2.log(format!("{}", err));
+                task_log!(worker2, "{}", err);
                 errors2.fetch_add(1, Ordering::SeqCst);
-                rename_corrupted_chunk(datastore2.clone(), &digest, worker2.clone());
+                rename_corrupted_chunk(datastore2.clone(), &digest, &worker2);
             } else {
                 verified_chunks2.lock().unwrap().insert(digest);
             }
@@ -134,7 +139,7 @@ fn verify_index_chunks(
 
     for pos in 0..index.index_count() {
 
-        worker.fail_on_abort()?;
+        worker.check_abort()?;
         crate::tools::fail_on_shutdown()?;
 
         let info = index.chunk_info(pos).unwrap();
@@ -146,7 +151,7 @@ fn verify_index_chunks(
 
         if corrupt_chunks.lock().unwrap().contains(&info.digest) {
             let digest_str = proxmox::tools::digest_to_hex(&info.digest);
-            worker.log(format!("chunk {} was marked as corrupt", digest_str));
+            task_log!(worker, "chunk {} was marked as corrupt", digest_str);
             errors.fetch_add(1, Ordering::SeqCst);
             continue;
         }
@@ -154,9 +159,9 @@ fn verify_index_chunks(
         match datastore.load_chunk(&info.digest) {
             Err(err) => {
                 corrupt_chunks.lock().unwrap().insert(info.digest);
-                worker.log(format!("can't verify chunk, load failed - {}", err));
+                task_log!(worker, "can't verify chunk, load failed - {}", err);
                 errors.fetch_add(1, Ordering::SeqCst);
-                rename_corrupted_chunk(datastore.clone(), &info.digest, worker.clone());
+                rename_corrupted_chunk(datastore.clone(), &info.digest, &worker);
                 continue;
             }
             Ok(chunk) => {
@@ -179,8 +184,16 @@ fn verify_index_chunks(
 
     let error_count = errors.load(Ordering::SeqCst);
 
-    worker.log(format!("  verified {:.2}/{:.2} MiB in {:.2} seconds, speed {:.2}/{:.2} MiB/s ({} errors)",
-                       read_bytes_mib, decoded_bytes_mib, elapsed, read_speed, decode_speed, error_count));
+    task_log!(
+        worker,
+        "  verified {:.2}/{:.2} MiB in {:.2} seconds, speed {:.2}/{:.2} MiB/s ({} errors)",
+        read_bytes_mib,
+        decoded_bytes_mib,
+        elapsed,
+        read_speed,
+        decode_speed,
+        error_count,
+    );
 
     if errors.load(Ordering::SeqCst) > 0 {
         bail!("chunks could not be verified");
@@ -195,7 +208,7 @@ fn verify_fixed_index(
     info: &FileInfo,
     verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
     corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
-    worker: Arc<WorkerTask>,
+    worker: Arc<dyn TaskState + Send + Sync>,
 ) -> Result<(), Error> {
 
     let mut path = backup_dir.relative_path();
@@ -212,7 +225,14 @@ fn verify_fixed_index(
         bail!("wrong index checksum");
     }
 
-    verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, info.chunk_crypt_mode(), worker)
+    verify_index_chunks(
+        datastore,
+        Box::new(index),
+        verified_chunks,
+        corrupt_chunks,
+        info.chunk_crypt_mode(),
+        worker,
+    )
 }
 
 fn verify_dynamic_index(
@@ -221,7 +241,7 @@ fn verify_dynamic_index(
     info: &FileInfo,
     verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
     corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
-    worker: Arc<WorkerTask>,
+    worker: Arc<dyn TaskState + Send + Sync>,
 ) -> Result<(), Error> {
 
     let mut path = backup_dir.relative_path();
@@ -238,7 +258,14 @@ fn verify_dynamic_index(
         bail!("wrong index checksum");
     }
 
-    verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, info.chunk_crypt_mode(), worker)
+    verify_index_chunks(
+        datastore,
+        Box::new(index),
+        verified_chunks,
+        corrupt_chunks,
+        info.chunk_crypt_mode(),
+        worker,
+    )
 }
 
 /// Verify a single backup snapshot
@@ -255,25 +282,68 @@ pub fn verify_backup_dir(
     backup_dir: &BackupDir,
     verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
     corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
-    worker: Arc<WorkerTask>
+    worker: Arc<dyn TaskState + Send + Sync>,
+    upid: UPID,
 ) -> Result<bool, Error> {
+    let snap_lock = lock_dir_noblock_shared(
+        &datastore.snapshot_path(&backup_dir),
+        "snapshot",
+        "locked by another operation");
+    match snap_lock {
+        Ok(snap_lock) => verify_backup_dir_with_lock(
+            datastore,
+            backup_dir,
+            verified_chunks,
+            corrupt_chunks,
+            worker,
+            upid,
+            snap_lock
+        ),
+        Err(err) => {
+            task_log!(
+                worker,
+                "SKIPPED: verify {}:{} - could not acquire snapshot lock: {}",
+                datastore.name(),
+                backup_dir,
+                err,
+            );
+            Ok(true)
+        }
+    }
+}
 
-    let mut manifest = match datastore.load_manifest(&backup_dir) {
+/// See verify_backup_dir
+pub fn verify_backup_dir_with_lock(
+    datastore: Arc<DataStore>,
+    backup_dir: &BackupDir,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    worker: Arc<dyn TaskState + Send + Sync>,
+    upid: UPID,
+    _snap_lock: Dir,
+) -> Result<bool, Error> {
+    let manifest = match datastore.load_manifest(&backup_dir) {
         Ok((manifest, _)) => manifest,
         Err(err) => {
-            worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err));
+            task_log!(
+                worker,
+                "verify {}:{} - manifest load error: {}",
+                datastore.name(),
+                backup_dir,
+                err,
+            );
             return Ok(false);
         }
     };
 
-    worker.log(format!("verify {}:{}", datastore.name(), backup_dir));
+    task_log!(worker, "verify {}:{}", datastore.name(), backup_dir);
 
     let mut error_count = 0;
 
     let mut verify_result = VerifyState::Ok;
     for info in manifest.files() {
         let result = proxmox::try_block!({
-            worker.log(format!("  check {}", info.filename));
+            task_log!(worker, "  check {}", info.filename);
             match archive_type(&info.filename)? {
                 ArchiveType::FixedIndex =>
                     verify_fixed_index(
@@ -297,11 +367,18 @@ pub fn verify_backup_dir(
             }
         });
 
-        worker.fail_on_abort()?;
+        worker.check_abort()?;
         crate::tools::fail_on_shutdown()?;
 
         if let Err(err) = result {
-            worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err));
+            task_log!(
+                worker,
+                "verify {}:{}/{} failed: {}",
+                datastore.name(),
+                backup_dir,
+                info.filename,
+                err,
+            );
             error_count += 1;
             verify_result = VerifyState::Failed;
         }
@@ -310,11 +387,12 @@ pub fn verify_backup_dir(
 
     let verify_state = SnapshotVerifyState {
         state: verify_result,
-        upid: worker.upid().clone(),
+        upid,
     };
-    manifest.unprotected["verify_state"] = serde_json::to_value(verify_state)?;
-    datastore.store_manifest(&backup_dir, serde_json::to_value(manifest)?)
-        .map_err(|err| format_err!("unable to store manifest blob - {}", err))?;
+    let verify_state = serde_json::to_value(verify_state)?;
+    datastore.update_manifest(&backup_dir, |manifest| {
+        manifest.unprotected["verify_state"] = verify_state;
+    }).map_err(|err| format_err!("unable to update manifest blob - {}", err))?;
 
     Ok(error_count == 0)
 }
@@ -332,19 +410,27 @@ pub fn verify_backup_group(
     verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
     corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
     progress: Option<(usize, usize)>, // (done, snapshot_count)
-    worker: Arc<WorkerTask>,
+    worker: Arc<dyn TaskState + Send + Sync>,
+    upid: &UPID,
+    filter: &dyn Fn(&BackupInfo) -> bool,
 ) -> Result<(usize, Vec<String>), Error> {
 
     let mut errors = Vec::new();
     let mut list = match group.list_backups(&datastore.base_path()) {
         Ok(list) => list,
         Err(err) => {
-            worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err));
+            task_log!(
+                worker,
+                "verify group {}:{} - unable to list backups: {}",
+                datastore.name(),
+                group,
+                err,
+            );
             return Ok((0, errors));
         }
     };
 
-    worker.log(format!("verify group {}:{}", datastore.name(), group));
+    task_log!(worker, "verify group {}:{}", datastore.name(), group);
 
     let (done, snapshot_count) = progress.unwrap_or((0, list.len()));
 
@@ -352,13 +438,37 @@ pub fn verify_backup_group(
     BackupInfo::sort_list(&mut list, false); // newest first
     for info in list {
         count += 1;
-        if !verify_backup_dir(datastore.clone(), &info.backup_dir, verified_chunks.clone(), corrupt_chunks.clone(), worker.clone())?{
+
+        if filter(&info) == false {
+            task_log!(
+                worker,
+                "SKIPPED: verify {}:{} (recently verified)",
+                datastore.name(),
+                info.backup_dir,
+            );
+            continue;
+        }
+
+        if !verify_backup_dir(
+            datastore.clone(),
+            &info.backup_dir,
+            verified_chunks.clone(),
+            corrupt_chunks.clone(),
+            worker.clone(),
+            upid.clone(),
+        )? {
             errors.push(info.backup_dir.to_string());
         }
         if snapshot_count != 0 {
             let pos = done + count;
             let percentage = ((pos as f64) * 100.0)/(snapshot_count as f64);
-            worker.log(format!("percentage done: {:.2}% ({} of {} snapshots)", percentage, pos, snapshot_count));
+            task_log!(
+                worker,
+                "percentage done: {:.2}% ({} of {} snapshots)",
+                percentage,
+                pos,
+                snapshot_count,
+            );
         }
     }
 
@@ -372,8 +482,12 @@ pub fn verify_backup_group(
 /// Returns
 /// - Ok(failed_dirs) where failed_dirs had verification errors
 /// - Err(_) if task was aborted
-pub fn verify_all_backups(datastore: Arc<DataStore>, worker: Arc<WorkerTask>) -> Result<Vec<String>, Error> {
-
+pub fn verify_all_backups(
+    datastore: Arc<DataStore>,
+    worker: Arc<dyn TaskState + Send + Sync>,
+    upid: &UPID,
+    filter: &dyn Fn(&BackupInfo) -> bool,
+) -> Result<Vec<String>, Error> {
     let mut errors = Vec::new();
 
     let mut list = match BackupGroup::list_groups(&datastore.base_path()) {
@@ -382,7 +496,12 @@ pub fn verify_all_backups(datastore: Arc<DataStore>, worker: Arc<WorkerTask>) ->
             .filter(|group| !(group.backup_type() == "host" && group.backup_id() == "benchmark"))
             .collect::<Vec<BackupGroup>>(),
         Err(err) => {
-            worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
+            task_log!(
+                worker,
+                "verify datastore {} - unable to list backups: {}",
+                datastore.name(),
+                err,
+            );
             return Ok(errors);
         }
     };
@@ -400,7 +519,7 @@ pub fn verify_all_backups(datastore: Arc<DataStore>, worker: Arc<WorkerTask>) ->
     // start with 64 chunks since we assume there are few corrupt ones
     let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
 
-    worker.log(format!("verify datastore {} ({} snapshots)", datastore.name(), snapshot_count));
+    task_log!(worker, "verify datastore {} ({} snapshots)", datastore.name(), snapshot_count);
 
     let mut done = 0;
     for group in list {
@@ -411,6 +530,8 @@ pub fn verify_all_backups(datastore: Arc<DataStore>, worker: Arc<WorkerTask>) ->
             corrupt_chunks.clone(),
             Some((done, snapshot_count)),
             worker.clone(),
+            upid,
+            filter,
         )?;
         errors.append(&mut group_errors);
 

src/bin/proxmox-backup-api.rs

@@ -37,7 +37,7 @@ async fn run() -> Result<(), Error> {
     config::update_self_signed_cert(false)?;
 
     proxmox_backup::rrd::create_rrdb_dir()?;
-    proxmox_backup::config::jobstate::create_jobstate_dir()?;
+    proxmox_backup::server::jobstate::create_jobstate_dir()?;
 
     if let Err(err) = generate_auth_key() {
         bail!("unable to generate auth key - {}", err);
@@ -49,9 +49,11 @@ async fn run() -> Result<(), Error> {
     }
     let _ = csrf_secret(); // load with lazy_static
 
-    let config = server::ApiConfig::new(
+    let mut config = server::ApiConfig::new(
         buildcfg::JS_DIR, &proxmox_backup::api2::ROUTER, RpcEnvironmentType::PRIVILEGED)?;
 
+    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN)?;
+
     let rest_server = RestServer::new(config);
 
     // http server future:

src/bin/proxmox-backup-client.rs

@@ -36,6 +36,7 @@ use proxmox_backup::api2::types::*;
 use proxmox_backup::api2::version;
 use proxmox_backup::client::*;
 use proxmox_backup::pxar::catalog::*;
+use proxmox_backup::config::user::complete_user_name;
 use proxmox_backup::backup::{
     archive_type,
     decrypt_key,
@@ -412,6 +413,45 @@ async fn list_backup_groups(param: Value) -> Result<Value, Error> {
     Ok(Value::Null)
 }
 
+#[api(
+   input: {
+        properties: {
+            repository: {
+                schema: REPO_URL_SCHEMA,
+                optional: true,
+            },
+            group: {
+                type: String,
+                description: "Backup group.",
+            },
+            "new-owner": {
+                type: Userid,
+            },
+        }
+   }
+)]
+/// Change owner of a backup group
+async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Error> {
+
+    let repo = extract_repository_from_value(&param)?;
+
+    let mut client = connect(repo.host(), repo.port(), repo.user())?;
+
+    param.as_object_mut().unwrap().remove("repository");
+
+    let group: BackupGroup = group.parse()?;
+
+    param["backup-type"] = group.backup_type().into();
+    param["backup-id"] = group.backup_id().into();
+
+    let path = format!("api2/json/admin/datastore/{}/change-owner", repo.store());
+    client.post(&path, Some(param)).await?;
+
+    record_repository(&repo);
+
+    Ok(())
+}
+
 #[api(
    input: {
         properties: {
@@ -470,7 +510,7 @@ async fn list_snapshots(param: Value) -> Result<Value, Error> {
         .sortby("backup-id", false)
         .sortby("backup-time", false)
         .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
-        .column(ColumnConfig::new("size"))
+        .column(ColumnConfig::new("size").renderer(tools::format::render_bytes_human_readable))
         .column(ColumnConfig::new("files").renderer(render_files))
         ;
 
@@ -1817,17 +1857,29 @@ async fn complete_server_file_name_do(param: &HashMap<String, String>) -> Vec<St
 fn complete_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
     complete_server_file_name(arg, param)
         .iter()
-        .map(|v| tools::format::strip_server_file_expenstion(&v))
+        .map(|v| tools::format::strip_server_file_extension(&v))
         .collect()
 }
 
 pub fn complete_pxar_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
     complete_server_file_name(arg, param)
         .iter()
-        .filter_map(|v| {
-            let name = tools::format::strip_server_file_expenstion(&v);
-            if name.ends_with(".pxar") {
-                Some(name)
+        .filter_map(|name| {
+            if name.ends_with(".pxar.didx") {
+                Some(tools::format::strip_server_file_extension(name))
+            } else {
+                None
+            }
+        })
+        .collect()
+}
+
+pub fn complete_img_archive_name(arg: &str, param: &HashMap<String, String>) -> Vec<String> {
+    complete_server_file_name(arg, param)
+        .iter()
+        .filter_map(|name| {
+            if name.ends_with(".img.fidx") {
+                Some(tools::format::strip_server_file_extension(name))
             } else {
                 None
             }
@@ -1955,6 +2007,12 @@ fn main() {
     let version_cmd_def = CliCommand::new(&API_METHOD_API_VERSION)
         .completion_cb("repository", complete_repository);
 
+    let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
+        .arg_param(&["group", "new-owner"])
+        .completion_cb("group", complete_backup_group)
+        .completion_cb("new-owner",  complete_user_name)
+        .completion_cb("repository", complete_repository);
+
     let cmd_def = CliCommandMap::new()
         .insert("backup", backup_cmd_def)
         .insert("upload-log", upload_log_cmd_def)
@@ -1970,10 +2028,13 @@ fn main() {
         .insert("status", status_cmd_def)
         .insert("key", key::cli())
         .insert("mount", mount_cmd_def())
+        .insert("map", map_cmd_def())
+        .insert("unmap", unmap_cmd_def())
         .insert("catalog", catalog_mgmt_cli())
         .insert("task", task_mgmt_cli())
         .insert("version", version_cmd_def)
-        .insert("benchmark", benchmark_cmd_def);
+        .insert("benchmark", benchmark_cmd_def)
+        .insert("change-owner", change_owner_cmd_def);
 
     let rpcenv = CliEnvironment::new();
     run_cli_command(cmd_def, rpcenv, Some(|future| {

src/bin/proxmox-backup-proxy.rs

@@ -1,5 +1,6 @@
 use std::sync::{Arc};
 use std::path::{Path, PathBuf};
+use std::os::unix::io::AsRawFd;
 
 use anyhow::{bail, format_err, Error};
 use futures::*;
@@ -9,16 +10,45 @@ use openssl::ssl::{SslMethod, SslAcceptor, SslFiletype};
 use proxmox::try_block;
 use proxmox::api::RpcEnvironmentType;
 
+use proxmox_backup::{
+    backup::DataStore,
+    server::{
+        UPID,
+        WorkerTask,
+        ApiConfig,
+        rest::*,
+        jobstate::{
+            self,
+            Job,
+        },
+        rotate_task_log_archive,
+    },
+    tools::systemd::time::{
+        parse_calendar_event,
+        compute_next_event,
+    },
+};
+
+
 use proxmox_backup::api2::types::Userid;
 use proxmox_backup::configdir;
 use proxmox_backup::buildcfg;
 use proxmox_backup::server;
-use proxmox_backup::tools::daemon;
-use proxmox_backup::server::{ApiConfig, rest::*};
 use proxmox_backup::auth_helpers::*;
-use proxmox_backup::tools::disks::{ DiskManage, zfs_pool_stats };
+use proxmox_backup::tools::{
+    daemon,
+    disks::{
+        DiskManage,
+        zfs_pool_stats,
+    },
+    socket::{
+        set_tcp_keepalive,
+        PROXMOX_BACKUP_TCP_KEEPALIVE_TIME,
+    },
+};
 
 use proxmox_backup::api2::pull::do_sync_job;
+use proxmox_backup::server::do_verification_job;
 
 fn main() -> Result<(), Error> {
     proxmox_backup::tools::setup_safe_path_env();
@@ -63,13 +93,15 @@ async fn run() -> Result<(), Error> {
     config.register_template("index", &indexpath)?;
     config.register_template("console", "/usr/share/pve-xtermjs/index.html.hbs")?;
 
+    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN)?;
+
     let rest_server = RestServer::new(config);
 
     //openssl req -x509 -newkey rsa:4096 -keyout /etc/proxmox-backup/proxy.key -out /etc/proxmox-backup/proxy.pem -nodes
     let key_path = configdir!("/proxy.key");
     let cert_path = configdir!("/proxy.pem");
 
-    let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls()).unwrap();
+    let mut acceptor = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).unwrap();
     acceptor.set_private_key_file(key_path, SslFiletype::PEM)
         .map_err(|err| format_err!("unable to read proxy key {} - {}", key_path, err))?;
     acceptor.set_certificate_chain_file(cert_path)
@@ -87,6 +119,9 @@ async fn run() -> Result<(), Error> {
                     let acceptor = Arc::clone(&acceptor);
                     async move {
                         sock.set_nodelay(true).unwrap();
+
+                        let _ = set_tcp_keepalive(sock.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME);
+
                         Ok(tokio_openssl::accept(&acceptor, sock)
                             .await
                             .ok() // handshake errors aren't be fatal, so return None to filter
@@ -196,8 +231,8 @@ async fn schedule_tasks() -> Result<(), Error> {
 
     schedule_datastore_garbage_collection().await;
     schedule_datastore_prune().await;
-    schedule_datastore_verification().await;
     schedule_datastore_sync_jobs().await;
+    schedule_datastore_verify_jobs().await;
     schedule_task_log_rotate().await;
 
     Ok(())
@@ -205,14 +240,14 @@ async fn schedule_tasks() -> Result<(), Error> {
 
 async fn schedule_datastore_garbage_collection() {
 
-    use proxmox_backup::backup::DataStore;
-    use proxmox_backup::server::{UPID, WorkerTask};
     use proxmox_backup::config::{
-        jobstate::{self, Job},
-        datastore::{self, DataStoreConfig}
+        datastore::{
+            self,
+            DataStoreConfig,
+        },
     };
-    use proxmox_backup::tools::systemd::time::{
-        parse_calendar_event, compute_next_event};
+
+    let email = server::lookup_user_email(Userid::root_userid());
 
     let config = match datastore::config() {
         Err(err) => {
@@ -294,6 +329,7 @@ async fn schedule_datastore_garbage_collection() {
         };
 
         let store2 = store.clone();
+        let email2 = email.clone();
 
         if let Err(err) = WorkerTask::new_thread(
             worker_type,
@@ -306,7 +342,7 @@ async fn schedule_datastore_garbage_collection() {
                 worker.log(format!("starting garbage collection on store {}", store));
                 worker.log(format!("task triggered by schedule '{}'", event_str));
 
-                let result = datastore.garbage_collection(&worker);
+                let result = datastore.garbage_collection(&*worker, worker.upid());
 
                 let status = worker.create_state(&result);
 
@@ -314,6 +350,13 @@ async fn schedule_datastore_garbage_collection() {
                     eprintln!("could not finish job state for {}: {}", worker_type, err);
                 }
 
+                if let Some(email2) = email2 {
+                    let gc_status = datastore.last_gc_status();
+                    if let Err(err) = crate::server::send_gc_status(&email2, datastore.name(), &gc_status, &result) {
+                        eprintln!("send gc notification failed: {}", err);
+                    }
+                }
+
                 result
             }
         ) {
@@ -324,15 +367,17 @@ async fn schedule_datastore_garbage_collection() {
 
 async fn schedule_datastore_prune() {
 
-    use proxmox_backup::backup::{
-        PruneOptions, DataStore, BackupGroup, compute_prune_info};
-    use proxmox_backup::server::{WorkerTask};
-    use proxmox_backup::config::{
-        jobstate::{self, Job},
-        datastore::{self, DataStoreConfig}
+    use proxmox_backup::{
+        backup::{
+            PruneOptions,
+            BackupGroup,
+            compute_prune_info,
+        },
+        config::datastore::{
+            self,
+            DataStoreConfig,
+        },
     };
-    use proxmox_backup::tools::systemd::time::{
-        parse_calendar_event, compute_next_event};
 
     let config = match datastore::config() {
         Err(err) => {
@@ -469,125 +514,11 @@ async fn schedule_datastore_prune() {
     }
 }
 
-async fn schedule_datastore_verification() {
-    use proxmox_backup::backup::{DataStore, verify_all_backups};
-    use proxmox_backup::server::{WorkerTask};
-    use proxmox_backup::config::{
-        jobstate::{self, Job},
-        datastore::{self, DataStoreConfig}
-    };
-    use proxmox_backup::tools::systemd::time::{
-        parse_calendar_event, compute_next_event};
-
-    let config = match datastore::config() {
-        Err(err) => {
-            eprintln!("unable to read datastore config - {}", err);
-            return;
-        }
-        Ok((config, _digest)) => config,
-    };
-
-    for (store, (_, store_config)) in config.sections {
-        let datastore = match DataStore::lookup_datastore(&store) {
-            Ok(datastore) => datastore,
-            Err(err) => {
-                eprintln!("lookup_datastore failed - {}", err);
-                continue;
-            }
-        };
-
-        let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
-            Ok(c) => c,
-            Err(err) => {
-                eprintln!("datastore config from_value failed - {}", err);
-                continue;
-            }
-        };
-
-        let event_str = match store_config.verify_schedule {
-            Some(event_str) => event_str,
-            None => continue,
-        };
-
-        let event = match parse_calendar_event(&event_str) {
-            Ok(event) => event,
-            Err(err) => {
-                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
-                continue;
-            }
-        };
-
-        let worker_type = "verify";
-
-        let last = match jobstate::last_run_time(worker_type, &store) {
-            Ok(time) => time,
-            Err(err) => {
-                eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
-                continue;
-            }
-        };
-
-        let next = match compute_next_event(&event, last, false) {
-            Ok(Some(next)) => next,
-            Ok(None) => continue,
-            Err(err) => {
-                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
-                continue;
-            }
-        };
-
-        let now = proxmox::tools::time::epoch_i64();
-
-        if next > now { continue; }
-
-        let mut job = match Job::new(worker_type, &store) {
-            Ok(job) => job,
-            Err(_) => continue, // could not get lock
-        };
-
-        let worker_id = store.clone();
-        let store2 = store.clone();
-        if let Err(err) = WorkerTask::new_thread(
-            worker_type,
-            Some(worker_id),
-            Userid::backup_userid().clone(),
-            false,
-            move |worker| {
-                job.start(&worker.upid().to_string())?;
-                worker.log(format!("starting verification on store {}", store2));
-                worker.log(format!("task triggered by schedule '{}'", event_str));
-                let result = try_block!({
-                    let failed_dirs = verify_all_backups(datastore, worker.clone())?;
-                    if failed_dirs.len() > 0 {
-                        worker.log("Failed to verify following snapshots:");
-                        for dir in failed_dirs {
-                            worker.log(format!("\t{}", dir));
-                        }
-                        Err(format_err!("verification failed - please check the log for details"))
-                    } else {
-                        Ok(())
-                    }
-                });
-
-                let status = worker.create_state(&result);
-
-                if let Err(err) = job.finish(status) {
-                    eprintln!("could not finish job state for {}: {}", worker_type, err);
-                }
-
-                result
-            },
-        ) {
-            eprintln!("unable to start verification on store {} - {}", store, err);
-        }
-    }
-}
-
 async fn schedule_datastore_sync_jobs() {
 
-    use proxmox_backup::{
-        config::{ sync::{self, SyncJobConfig}, jobstate::{self, Job} },
-        tools::systemd::time::{ parse_calendar_event, compute_next_event },
+    use proxmox_backup::config::sync::{
+        self,
+        SyncJobConfig,
     };
 
     let config = match sync::config() {
@@ -656,17 +587,72 @@ async fn schedule_datastore_sync_jobs() {
     }
 }
 
-async fn schedule_task_log_rotate() {
-    use proxmox_backup::{
-        config::jobstate::{self, Job},
-        server::rotate_task_log_archive,
+async fn schedule_datastore_verify_jobs() {
+
+    use proxmox_backup::config::verify::{
+        self,
+        VerificationJobConfig,
     };
-    use proxmox_backup::server::WorkerTask;
-    use proxmox_backup::tools::systemd::time::{
-        parse_calendar_event, compute_next_event};
+
+    let config = match verify::config() {
+        Err(err) => {
+            eprintln!("unable to read verification job config - {}", err);
+            return;
+        }
+        Ok((config, _digest)) => config,
+    };
+    for (job_id, (_, job_config)) in config.sections {
+        let job_config: VerificationJobConfig = match serde_json::from_value(job_config) {
+            Ok(c) => c,
+            Err(err) => {
+                eprintln!("verification job config from_value failed - {}", err);
+                continue;
+            }
+        };
+        let event_str = match job_config.schedule {
+            Some(ref event_str) => event_str.clone(),
+            None => continue,
+        };
+        let event = match parse_calendar_event(&event_str) {
+            Ok(event) => event,
+            Err(err) => {
+                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
+                continue;
+            }
+        };
+        let worker_type = "verificationjob";
+        let last = match jobstate::last_run_time(worker_type, &job_id) {
+            Ok(time) => time,
+            Err(err) => {
+                eprintln!("could not get last run time of {} {}: {}", worker_type, job_id, err);
+                continue;
+            }
+        };
+        let next = match compute_next_event(&event, last, false) {
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
+            Err(err) => {
+                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
+                continue;
+            }
+        };
+        let now = proxmox::tools::time::epoch_i64();
+        if next > now { continue; }
+        let job = match Job::new(worker_type, &job_id) {
+            Ok(job) => job,
+            Err(_) => continue, // could not get lock
+        };
+        let userid = Userid::backup_userid().clone();
+        if let Err(err) = do_verification_job(job, job_config, &userid, Some(event_str)) {
+            eprintln!("unable to start datastore verification job {} - {}", &job_id, err);
+        }
+    }
+}
+
+async fn schedule_task_log_rotate() {
 
     let worker_type = "logrotate";
-    let job_id = "task-archive";
+    let job_id = "task_archive";
 
     let last = match jobstate::last_run_time(worker_type, job_id) {
         Ok(time) => time,
@@ -723,10 +709,11 @@ async fn schedule_task_log_rotate() {
         move |worker| {
             job.start(&worker.upid().to_string())?;
             worker.log(format!("starting task log rotation"));
-            // one entry has normally about ~100-150 bytes
-            let max_size = 500000; // at least 5000 entries
-            let max_files = 20; // at least 100000 entries
+
             let result = try_block!({
+                // rotate task log archive
+                let max_size = 500000; // a normal entry has about 100b, so ~ 5000 entries/file
+                let max_files = 20; // times twenty files gives at least 100000 task entries
                 let has_rotated = rotate_task_log_archive(max_size, true, Some(max_files))?;
                 if has_rotated {
                     worker.log(format!("task log archive was rotated"));

src/bin/proxmox_backup_client/benchmark.rs

@@ -86,7 +86,7 @@ struct BenchmarkResult {
 static BENCHMARK_RESULT_2020_TOP: BenchmarkResult =  BenchmarkResult {
     tls: Speed {
         speed: None,
-        top: 1_000_000.0 * 690.0, // TLS to localhost, AMD Ryzen 7 2700X
+        top: 1_000_000.0 * 1235.0, // TLS to localhost, AMD Ryzen 7 2700X
     },
     sha256: Speed {
         speed: None,

src/bin/proxmox_backup_client/key.rs

@@ -346,7 +346,7 @@ pub fn cli() -> CliCommandMap {
         .insert("create-master-key", key_create_master_key_cmd_def)
         .insert("import-master-pubkey", key_import_master_pubkey_cmd_def)
         .insert("change-passphrase", key_change_passphrase_cmd_def)
-        .insert("paper-key", paper_key_cmd_def)
+        .insert("paperkey", paper_key_cmd_def)
 }
 
 fn paperkey_html(data: &str, subject: Option<String>) -> Result<(), Error> {
@@ -414,13 +414,13 @@ fn paperkey_html(data: &str, subject: Option<String>) -> Result<(), Error> {
             println!("</p>");
 
             let data = data.join("\n");
-            let qr_code = generate_qr_code("png", data.as_bytes())?;
+            let qr_code = generate_qr_code("svg", data.as_bytes())?;
             let qr_code = base64::encode_config(&qr_code, base64::STANDARD_NO_PAD);
 
             println!("<center>");
             println!("<img");
             println!("width=\"{}pt\" height=\"{}pt\"", img_size_pt, img_size_pt);
-            println!("src=\"data:image/png;base64,{}\"/>", qr_code);
+            println!("src=\"data:image/svg+xml;base64,{}\"/>", qr_code);
             println!("</center>");
             println!("</div>");
        }
@@ -447,13 +447,13 @@ fn paperkey_html(data: &str, subject: Option<String>) -> Result<(), Error> {
 
     println!("</p>");
 
-    let qr_code = generate_qr_code("png", key_text.as_bytes())?;
+    let qr_code = generate_qr_code("svg", key_text.as_bytes())?;
     let qr_code = base64::encode_config(&qr_code, base64::STANDARD_NO_PAD);
 
     println!("<center>");
     println!("<img");
     println!("width=\"{}pt\" height=\"{}pt\"", img_size_pt, img_size_pt);
-    println!("src=\"data:image/png;base64,{}\"/>", qr_code);
+    println!("src=\"data:image/svg+xml;base64,{}\"/>", qr_code);
     println!("</center>");
 
     println!("</div>");

src/bin/proxmox_backup_client/mount.rs

@@ -3,6 +3,8 @@ use std::sync::Arc;
 use std::os::unix::io::RawFd;
 use std::path::Path;
 use std::ffi::OsStr;
+use std::collections::HashMap;
+use std::hash::BuildHasher;
 
 use anyhow::{bail, format_err, Error};
 use serde_json::Value;
@@ -10,6 +12,7 @@ use tokio::signal::unix::{signal, SignalKind};
 use nix::unistd::{fork, ForkResult, pipe};
 use futures::select;
 use futures::future::FutureExt;
+use futures::stream::{StreamExt, TryStreamExt};
 
 use proxmox::{sortable, identity};
 use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment, schema::*, cli::*};
@@ -23,6 +26,7 @@ use proxmox_backup::backup::{
     BackupDir,
     BackupGroup,
     BufferedDynamicReader,
+    AsyncIndexReader,
 };
 
 use proxmox_backup::client::*;
@@ -31,6 +35,7 @@ use crate::{
     REPO_URL_SCHEMA,
     extract_repository_from_value,
     complete_pxar_archive_name,
+    complete_img_archive_name,
     complete_group_or_snapshot,
     complete_repository,
     record_repository,
@@ -50,7 +55,37 @@ const API_METHOD_MOUNT: ApiMethod = ApiMethod::new(
             ("target", false, &StringSchema::new("Target directory path.").schema()),
             ("repository", true, &REPO_URL_SCHEMA),
             ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
-            ("verbose", true, &BooleanSchema::new("Verbose output.").default(false).schema()),
+            ("verbose", true, &BooleanSchema::new("Verbose output and stay in foreground.").default(false).schema()),
+        ]),
+    )
+);
+
+#[sortable]
+const API_METHOD_MAP: ApiMethod = ApiMethod::new(
+    &ApiHandler::Sync(&mount),
+    &ObjectSchema::new(
+        "Map a drive image from a VM backup to a local loopback device. Use 'unmap' to undo.
+WARNING: Only do this with *trusted* backups!",
+        &sorted!([
+            ("snapshot", false, &StringSchema::new("Group/Snapshot path.").schema()),
+            ("archive-name", false, &StringSchema::new("Backup archive name.").schema()),
+            ("repository", true, &REPO_URL_SCHEMA),
+            ("keyfile", true, &StringSchema::new("Path to encryption key.").schema()),
+            ("verbose", true, &BooleanSchema::new("Verbose output and stay in foreground.").default(false).schema()),
+        ]),
+    )
+);
+
+#[sortable]
+const API_METHOD_UNMAP: ApiMethod = ApiMethod::new(
+    &ApiHandler::Sync(&unmap),
+    &ObjectSchema::new(
+        "Unmap a loop device mapped with 'map' and release all resources.",
+        &sorted!([
+            ("name", true, &StringSchema::new(
+                concat!("Archive name, path to loopdev (/dev/loopX) or loop device number. ",
+                        "Omit to list all current mappings and force cleaning up leftover instances.")
+            ).schema()),
         ]),
     )
 );
@@ -65,6 +100,34 @@ pub fn mount_cmd_def() -> CliCommand {
         .completion_cb("target", tools::complete_file_name)
 }
 
+pub fn map_cmd_def() -> CliCommand {
+
+    CliCommand::new(&API_METHOD_MAP)
+        .arg_param(&["snapshot", "archive-name"])
+        .completion_cb("repository", complete_repository)
+        .completion_cb("snapshot", complete_group_or_snapshot)
+        .completion_cb("archive-name", complete_img_archive_name)
+}
+
+pub fn unmap_cmd_def() -> CliCommand {
+
+    CliCommand::new(&API_METHOD_UNMAP)
+        .arg_param(&["name"])
+        .completion_cb("name", complete_mapping_names)
+}
+
+fn complete_mapping_names<S: BuildHasher>(_arg: &str, _param: &HashMap<String, String, S>)
+    -> Vec<String>
+{
+    match tools::fuse_loop::find_all_mappings() {
+        Ok(mappings) => mappings
+            .filter_map(|(name, _)| {
+                tools::systemd::unescape_unit(&name).ok()
+            }).collect(),
+        Err(_) => Vec::new()
+    }
+}
+
 fn mount(
     param: Value,
     _info: &ApiMethod,
@@ -81,7 +144,7 @@ fn mount(
     // Process should be deamonized.
     // Make sure to fork before the async runtime is instantiated to avoid troubles.
     let pipe = pipe()?;
-    match fork() {
+    match unsafe { fork() } {
         Ok(ForkResult::Parent { .. }) => {
             nix::unistd::close(pipe.1).unwrap();
             // Blocks the parent process until we are ready to go in the child
@@ -100,9 +163,10 @@ fn mount(
 async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let archive_name = tools::required_string_param(&param, "archive-name")?;
-    let target = tools::required_string_param(&param, "target")?;
     let client = connect(repo.host(), repo.port(), repo.user())?;
 
+    let target = param["target"].as_str();
+
     record_repository(&repo);
 
     let path = tools::required_string_param(&param, "snapshot")?;
@@ -124,9 +188,17 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
     };
 
     let server_archive_name = if archive_name.ends_with(".pxar") {
+        if let None = target {
+            bail!("use the 'mount' command to mount pxar archives");
+        }
         format!("{}.didx", archive_name)
+    } else if archive_name.ends_with(".img") {
+        if let Some(_) = target {
+            bail!("use the 'map' command to map drive images");
+        }
+        format!("{}.fidx", archive_name)
     } else {
-        bail!("Can only mount pxar archives.");
+        bail!("Can only mount/map pxar archives and drive images.");
     };
 
     let client = BackupReader::start(
@@ -143,25 +215,7 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
 
     let file_info = manifest.lookup_file_info(&server_archive_name)?;
 
-    if server_archive_name.ends_with(".didx") {
-        let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
-        let most_used = index.find_most_used_chunks(8);
-        let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
-        let reader = BufferedDynamicReader::new(index, chunk_reader);
-        let archive_size = reader.archive_size();
-        let reader: proxmox_backup::pxar::fuse::Reader =
-            Arc::new(BufferedDynamicReadAt::new(reader));
-        let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
-        let options = OsStr::new("ro,default_permissions");
-
-        let session = proxmox_backup::pxar::fuse::Session::mount(
-            decoder,
-            &options,
-            false,
-            Path::new(target),
-        )
-        .map_err(|err| format_err!("pxar mount failed: {}", err))?;
-
+    let daemonize = || -> Result<(), Error> {
         if let Some(pipe) = pipe {
             nix::unistd::chdir(Path::new("/")).unwrap();
             // Finish creation of daemon by redirecting filedescriptors.
@@ -182,15 +236,132 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
             nix::unistd::close(pipe).unwrap();
         }
 
-        let mut interrupt = signal(SignalKind::interrupt())?;
+        Ok(())
+    };
+
+    let options = OsStr::new("ro,default_permissions");
+
+    // handle SIGINT and SIGTERM
+    let mut interrupt_int = signal(SignalKind::interrupt())?;
+    let mut interrupt_term = signal(SignalKind::terminate())?;
+    let mut interrupt = futures::future::select(interrupt_int.next(), interrupt_term.next());
+
+    if server_archive_name.ends_with(".didx") {
+        let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;
+        let most_used = index.find_most_used_chunks(8);
+        let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
+        let reader = BufferedDynamicReader::new(index, chunk_reader);
+        let archive_size = reader.archive_size();
+        let reader: proxmox_backup::pxar::fuse::Reader =
+            Arc::new(BufferedDynamicReadAt::new(reader));
+        let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
+
+        let session = proxmox_backup::pxar::fuse::Session::mount(
+            decoder,
+            &options,
+            false,
+            Path::new(target.unwrap()),
+        )
+        .map_err(|err| format_err!("pxar mount failed: {}", err))?;
+
+        daemonize()?;
+
         select! {
             res = session.fuse() => res?,
-            _ = interrupt.recv().fuse() => {
+            _ = interrupt => {
                 // exit on interrupted
             }
         }
+    } else if server_archive_name.ends_with(".fidx") {
+        let index = client.download_fixed_index(&manifest, &server_archive_name).await?;
+        let size = index.index_bytes();
+        let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), HashMap::new());
+        let reader = AsyncIndexReader::new(index, chunk_reader);
+
+        let name = &format!("{}:{}/{}", repo.to_string(), path, archive_name);
+        let name_escaped = tools::systemd::escape_unit(name, false);
+
+        let mut session = tools::fuse_loop::FuseLoopSession::map_loop(size, reader, &name_escaped, options).await?;
+        let loopdev = session.loopdev_path.clone();
+
+        let (st_send, st_recv) = futures::channel::mpsc::channel(1);
+        let (mut abort_send, abort_recv) = futures::channel::mpsc::channel(1);
+        let mut st_recv = st_recv.fuse();
+        let mut session_fut = session.main(st_send, abort_recv).boxed().fuse();
+
+        // poll until loop file is mapped (or errors)
+        select! {
+            res = session_fut => {
+                bail!("FUSE session unexpectedly ended before loop file mapping");
+            },
+            res = st_recv.try_next() => {
+                if let Err(err) = res {
+                    // init went wrong, abort now
+                    abort_send.try_send(()).map_err(|err|
+                        format_err!("error while sending abort signal - {}", err))?;
+                    // ignore and keep original error cause
+                    let _ = session_fut.await;
+                    return Err(err);
+                }
+            }
+        }
+
+        // daemonize only now to be able to print mapped loopdev or startup errors
+        println!("Image '{}' mapped on {}", name, loopdev);
+        daemonize()?;
+
+        // continue polling until complete or interrupted (which also happens on unmap)
+        select! {
+            res = session_fut => res?,
+            _ = interrupt => {
+                // exit on interrupted
+                abort_send.try_send(()).map_err(|err|
+                    format_err!("error while sending abort signal - {}", err))?;
+                session_fut.await?;
+            }
+        }
+
+        println!("Image unmapped");
     } else {
-        bail!("unknown archive file extension (expected .pxar)");
+        bail!("unknown archive file extension (expected .pxar or .img)");
+    }
+
+    Ok(Value::Null)
+}
+
+fn unmap(
+    param: Value,
+    _info: &ApiMethod,
+    _rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Value, Error> {
+
+    let mut name = match param["name"].as_str() {
+        Some(name) => name.to_owned(),
+        None => {
+            tools::fuse_loop::cleanup_unused_run_files(None);
+            let mut any = false;
+            for (backing, loopdev) in tools::fuse_loop::find_all_mappings()? {
+                let name = tools::systemd::unescape_unit(&backing)?;
+                println!("{}:\t{}", loopdev.unwrap_or("(unmapped)".to_owned()), name);
+                any = true;
+            }
+            if !any {
+                println!("Nothing mapped.");
+            }
+            return Ok(Value::Null);
+        },
+    };
+
+    // allow loop device number alone
+    if let Ok(num) = name.parse::<u8>() {
+        name = format!("/dev/loop{}", num);
+    }
+
+    if name.starts_with("/dev/loop") {
+        tools::fuse_loop::unmap_loopdev(name)?;
+    } else {
+        let name = tools::systemd::escape_unit(&name, false);
+        tools::fuse_loop::unmap_name(name)?;
     }
 
     Ok(Value::Null)

src/buildcfg.rs

@@ -4,6 +4,8 @@
 pub const CONFIGDIR: &str = "/etc/proxmox-backup";
 pub const JS_DIR: &str = "/usr/share/javascript/proxmox-backup";
 
+pub const API_ACCESS_LOG_FN: &str = "/var/log/proxmox-backup/api/access.log";
+
 /// Prepend configuration directory to a file name
 ///
 /// This is a simply way to get the full path for configuration files.

src/client/backup_writer.rs

@@ -38,6 +38,9 @@ pub struct BackupStats {
     pub csum: [u8; 32],
 }
 
+type UploadQueueSender = mpsc::Sender<(MergedChunkInfo, Option<h2::client::ResponseFuture>)>;
+type UploadResultReceiver = oneshot::Receiver<Result<(), Error>>;
+
 impl BackupWriter {
 
     fn new(h2: H2Client, abort: AbortHandle, crypt_config: Option<Arc<CryptConfig>>, verbose: bool) -> Arc<Self> {
@@ -262,7 +265,7 @@ impl BackupWriter {
         let archive = if self.verbose {
             archive_name.to_string()
         } else {
-            crate::tools::format::strip_server_file_expenstion(archive_name.clone())
+            crate::tools::format::strip_server_file_extension(archive_name)
         };
         if archive_name != CATALOG_NAME {
             let speed: HumanByte = ((uploaded * 1_000_000) / (duration.as_micros() as usize)).into();
@@ -335,15 +338,15 @@ impl BackupWriter {
         (verify_queue_tx, verify_result_rx)
     }
 
-    fn append_chunk_queue(h2: H2Client, wid: u64, path: String, verbose: bool) -> (
-        mpsc::Sender<(MergedChunkInfo, Option<h2::client::ResponseFuture>)>,
-        oneshot::Receiver<Result<(), Error>>,
-    ) {
+    fn append_chunk_queue(
+        h2: H2Client,
+        wid: u64,
+        path: String,
+        verbose: bool,
+    ) -> (UploadQueueSender, UploadResultReceiver) {
         let (verify_queue_tx, verify_queue_rx) = mpsc::channel(64);
         let (verify_result_tx, verify_result_rx) = oneshot::channel();
 
-        let h2_2 = h2.clone();
-
         // FIXME: async-block-ify this code!
         tokio::spawn(
             verify_queue_rx
@@ -381,7 +384,7 @@ impl BackupWriter {
                             let request = H2Client::request_builder("localhost", "PUT", &path, None, Some("application/json")).unwrap();
                             let param_data = bytes::Bytes::from(param.to_string().into_bytes());
                             let upload_data = Some(param_data);
-                            h2_2.send_request(request, upload_data)
+                            h2.send_request(request, upload_data)
                                 .and_then(move |response| {
                                     response
                                         .map_err(Error::from)
@@ -489,6 +492,10 @@ impl BackupWriter {
         Ok(manifest)
     }
 
+    // We have no `self` here for `h2` and `verbose`, the only other arg "common" with 1 other
+    // funciton in the same path is `wid`, so those 3 could be in a struct, but there's no real use
+    // since this is a private method.
+    #[allow(clippy::too_many_arguments)]
     fn upload_chunk_info_stream(
         h2: H2Client,
         wid: u64,
@@ -515,7 +522,7 @@ impl BackupWriter {
         let is_fixed_chunk_size = prefix == "fixed";
 
         let (upload_queue, upload_result) =
-            Self::append_chunk_queue(h2.clone(), wid, append_chunk_path.to_owned(), verbose);
+            Self::append_chunk_queue(h2.clone(), wid, append_chunk_path, verbose);
 
         let start_time = std::time::Instant::now();
 
@@ -574,10 +581,12 @@ impl BackupWriter {
                     let digest = chunk_info.digest;
                     let digest_str = digest_to_hex(&digest);
 
-                    if false && verbose { // TO verbose, needs finer verbosity setting granularity
+                    /* too verbose, needs finer verbosity setting granularity
+                    if verbose {
                         println!("upload new chunk {} ({} bytes, offset {})", digest_str,
                                  chunk_info.chunk_len, offset);
                     }
+                    */
 
                     let chunk_data = chunk_info.chunk.into_inner();
                     let param = json!({

src/client/http_client.rs

@@ -1,5 +1,4 @@
 use std::io::Write;
-use std::task::{Context, Poll};
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::Duration;
 
@@ -18,15 +17,17 @@ use xdg::BaseDirectories;
 use proxmox::{
     api::error::HttpError,
     sys::linux::tty,
-    tools::{
-        fs::{file_get_json, replace_file, CreateOptions},
-    }
+    tools::fs::{file_get_json, replace_file, CreateOptions},
 };
 
 use super::pipe_to_stream::PipeToSendStream;
 use crate::api2::types::Userid;
-use crate::tools::async_io::EitherStream;
-use crate::tools::{self, BroadcastFuture, DEFAULT_ENCODE_SET};
+use crate::tools::{
+    self,
+    BroadcastFuture,
+    DEFAULT_ENCODE_SET,
+    http::HttpsConnector,
+};
 
 #[derive(Clone)]
 pub struct AuthInfo {
@@ -181,10 +182,8 @@ fn load_fingerprint(prefix: &str, server: &str) -> Option<String> {
 
     for line in raw.split('\n') {
         let items: Vec<String> = line.split_whitespace().map(String::from).collect();
-        if items.len() == 2 {
-            if &items[0] == server {
-                return Some(items[1].clone());
-            }
+        if items.len() == 2 && &items[0] == server {
+            return Some(items[1].clone());
         }
     }
 
@@ -212,11 +211,11 @@ fn store_ticket_info(prefix: &str, server: &str, username: &str, ticket: &str, t
 
     let empty = serde_json::map::Map::new();
     for (server, info) in data.as_object().unwrap_or(&empty) {
-        for (_user, uinfo) in info.as_object().unwrap_or(&empty) {
+        for (user, uinfo) in info.as_object().unwrap_or(&empty) {
             if let Some(timestamp) = uinfo["timestamp"].as_i64() {
                 let age = now - timestamp;
                 if age < ticket_lifetime {
-                    new_data[server][username] = uinfo.clone();
+                    new_data[server][user] = uinfo.clone();
                 }
             }
         }
@@ -294,10 +293,11 @@ impl HttpClient {
             ssl_connector_builder.set_verify(openssl::ssl::SslVerifyMode::NONE);
         }
 
-        let mut httpc = hyper::client::HttpConnector::new();
+        let mut httpc = HttpConnector::new();
         httpc.set_nodelay(true); // important for h2 download performance!
         httpc.enforce_http(false); // we want https...
 
+        httpc.set_connect_timeout(Some(std::time::Duration::new(10, 0)));
         let https = HttpsConnector::with_connector(httpc, ssl_connector_builder.build());
 
         let client = Client::builder()
@@ -609,7 +609,7 @@ impl HttpClient {
             .await?;
 
         let connection = connection
-            .map_err(|_| panic!("HTTP/2.0 connection failed"));
+            .map_err(|_| eprintln!("HTTP/2.0 connection failed"));
 
         let (connection, abort) = futures::future::abortable(connection);
         // A cancellable future returns an Option which is None when cancelled and
@@ -921,61 +921,3 @@ impl H2Client {
         }
     }
 }
-
-#[derive(Clone)]
-pub struct HttpsConnector {
-    http: HttpConnector,
-    ssl_connector: std::sync::Arc<SslConnector>,
-}
-
-impl HttpsConnector {
-    pub fn with_connector(mut http: HttpConnector, ssl_connector: SslConnector) -> Self {
-        http.enforce_http(false);
-
-        Self {
-            http,
-            ssl_connector: std::sync::Arc::new(ssl_connector),
-        }
-    }
-}
-
-type MaybeTlsStream = EitherStream<
-    tokio::net::TcpStream,
-    tokio_openssl::SslStream<tokio::net::TcpStream>,
->;
-
-impl hyper::service::Service<Uri> for HttpsConnector {
-    type Response = MaybeTlsStream;
-    type Error = Error;
-    type Future = std::pin::Pin<Box<
-        dyn Future<Output = Result<Self::Response, Self::Error>> + Send + 'static
-    >>;
-
-    fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        // This connector is always ready, but others might not be.
-        Poll::Ready(Ok(()))
-    }
-
-    fn call(&mut self, dst: Uri) -> Self::Future {
-        let mut this = self.clone();
-        async move {
-            let is_https = dst
-                .scheme()
-                .ok_or_else(|| format_err!("missing URL scheme"))?
-                == "https";
-            let host = dst
-                .host()
-                .ok_or_else(|| format_err!("missing hostname in destination url?"))?
-                .to_string();
-
-            let config = this.ssl_connector.configure();
-            let conn = this.http.call(dst).await?;
-            if is_https {
-                let conn = tokio_openssl::connect(config?, &host, conn).await?;
-                Ok(MaybeTlsStream::Right(conn))
-            } else {
-                Ok(MaybeTlsStream::Left(conn))
-            }
-        }.boxed()
-    }
-}

src/client/pull.rs

@@ -251,8 +251,8 @@ async fn pull_snapshot(
         Err(err) => {
             match err.downcast_ref::<HttpError>() {
                 Some(HttpError { code, message }) => {
-                    match code {
-                        &StatusCode::NOT_FOUND => {
+                    match *code {
+                        StatusCode::NOT_FOUND => {
                             worker.log(format!("skipping snapshot {} - vanished since start of sync", snapshot));
                             return Ok(());
                         },
@@ -434,7 +434,7 @@ pub async fn pull_group(
         let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
 
         // in-progress backups can't be synced
-        if let None = item.size {
+        if item.size.is_none() {
             worker.log(format!("skipping snapshot {} - in-progress backup", snapshot));
             continue;
         }
@@ -520,9 +520,8 @@ pub async fn pull_store(
     }
 
     let group_count = list.len();
-    let mut groups_done = 0;
 
-    for item in list {
+    for (groups_done, item) in list.into_iter().enumerate() {
         let group = BackupGroup::new(&item.backup_type, &item.backup_id);
 
         let (owner, _lock_guard) = tgt_store.create_locked_backup_group(&group, &userid)?;
@@ -532,22 +531,23 @@ pub async fn pull_store(
                                item.backup_type, item.backup_id, userid, owner));
             errors = true; // do not stop here, instead continue
 
-        } else {
-
-            if let Err(err) = pull_group(
-                worker,
-                client,
-                src_repo,
-                tgt_store.clone(),
-                &group,
-                delete,
-                Some((groups_done, group_count)),
-            ).await {
-                worker.log(format!("sync group {}/{} failed - {}", item.backup_type, item.backup_id, err));
-                errors = true; // do not stop here, instead continue
-            }
+        } else if let Err(err) = pull_group(
+            worker,
+            client,
+            src_repo,
+            tgt_store.clone(),
+            &group,
+            delete,
+            Some((groups_done, group_count)),
+        ).await {
+            worker.log(format!(
+                "sync group {}/{} failed - {}",
+                item.backup_type,
+                item.backup_id,
+                err,
+            ));
+            errors = true; // do not stop here, instead continue
         }
-        groups_done += 1;
     }
 
     if delete {

src/client/pxar_backup_stream.rs

@@ -1,7 +1,7 @@
 use std::collections::HashSet;
 use std::io::Write;
 //use std::os::unix::io::FromRawFd;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::pin::Pin;
 use std::sync::{Arc, Mutex};
 use std::task::{Context, Poll};
@@ -38,9 +38,8 @@ impl Drop for PxarBackupStream {
 impl PxarBackupStream {
     pub fn new<W: Write + Send + 'static>(
         dir: Dir,
-        _path: PathBuf,
         device_set: Option<HashSet<u64>>,
-        _verbose: bool,
+        verbose: bool,
         skip_lost_and_found: bool,
         catalog: Arc<Mutex<CatalogWriter<W>>>,
         patterns: Vec<MatchEntry>,
@@ -70,7 +69,12 @@ impl PxarBackupStream {
                         crate::pxar::Flags::DEFAULT,
                         device_set,
                         skip_lost_and_found,
-                        |_| Ok(()),
+                        |path| {
+                            if verbose {
+                                println!("{:?}", path);
+                            }
+                            Ok(())
+                        },
                         entries_max,
                         Some(&mut *catalog_guard),
                     ) {
@@ -97,11 +101,9 @@ impl PxarBackupStream {
         entries_max: usize,
     ) -> Result<Self, Error> {
         let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?;
-        let path = std::path::PathBuf::from(dirname);
 
         Self::new(
             dir,
-            path,
             device_set,
             verbose,
             skip_lost_and_found,

src/client/task_log.rs

@@ -43,8 +43,8 @@ pub async fn display_task_log(
             } else {
                 break;
             }
-        } else {
-            if lines != limit { bail!("got wrong number of lines from server ({} != {})", lines, limit); }
+        } else if lines != limit {
+            bail!("got wrong number of lines from server ({} != {})", lines, limit);
         }
     }
 

src/config.rs

@@ -18,11 +18,11 @@ use crate::buildcfg;
 pub mod acl;
 pub mod cached_user_info;
 pub mod datastore;
-pub mod jobstate;
 pub mod network;
 pub mod remote;
 pub mod sync;
 pub mod user;
+pub mod verify;
 
 /// Check configuration directory permissions
 ///

src/config/acl.rs

@@ -12,42 +12,47 @@ use ::serde::{Deserialize, Serialize};
 use serde::de::{value, IntoDeserializer};
 
 use proxmox::tools::{fs::replace_file, fs::CreateOptions};
-use proxmox::constnamemap;
+use proxmox::constnamedbitmap;
 use proxmox::api::{api, schema::*};
 
 use crate::api2::types::Userid;
 
 // define Privilege bitfield
 
-constnamemap! {
+constnamedbitmap! {
     /// Contains a list of Privileges
     PRIVILEGES: u64 => {
-        PRIV_SYS_AUDIT("Sys.Audit")                         = 1 << 0;
-        PRIV_SYS_MODIFY("Sys.Modify")                       = 1 << 1;
-        PRIV_SYS_POWER_MANAGEMENT("Sys.PowerManagement")    = 1 << 2;
+        PRIV_SYS_AUDIT("Sys.Audit");
+        PRIV_SYS_MODIFY("Sys.Modify");
+        PRIV_SYS_POWER_MANAGEMENT("Sys.PowerManagement");
 
-        PRIV_DATASTORE_AUDIT("Datastore.Audit")             = 1 << 3;
-        PRIV_DATASTORE_MODIFY("Datastore.Modify")           = 1 << 4;
-        PRIV_DATASTORE_READ("Datastore.Read")               = 1 << 5;
+        PRIV_DATASTORE_AUDIT("Datastore.Audit");
+        PRIV_DATASTORE_ALLOCATE("Datastore.Allocate");
+        PRIV_DATASTORE_MODIFY("Datastore.Modify");
+        PRIV_DATASTORE_READ("Datastore.Read");
 
         /// Datastore.Backup also requires backup ownership
-        PRIV_DATASTORE_BACKUP("Datastore.Backup")           = 1 << 6;
+        PRIV_DATASTORE_BACKUP("Datastore.Backup");
         /// Datastore.Prune also requires backup ownership
-        PRIV_DATASTORE_PRUNE("Datastore.Prune")             = 1 << 7;
+        PRIV_DATASTORE_PRUNE("Datastore.Prune");
 
-        PRIV_PERMISSIONS_MODIFY("Permissions.Modify")       = 1 << 8;
+        PRIV_PERMISSIONS_MODIFY("Permissions.Modify");
 
-        PRIV_REMOTE_AUDIT("Remote.Audit")                   = 1 << 9;
-        PRIV_REMOTE_MODIFY("Remote.Modify")                 = 1 << 10;
-        PRIV_REMOTE_READ("Remote.Read")                     = 1 << 11;
-        PRIV_REMOTE_PRUNE("Remote.Prune")                   = 1 << 12;
+        PRIV_REMOTE_AUDIT("Remote.Audit");
+        PRIV_REMOTE_MODIFY("Remote.Modify");
+        PRIV_REMOTE_READ("Remote.Read");
+        PRIV_REMOTE_PRUNE("Remote.Prune");
 
-        PRIV_SYS_CONSOLE("Sys.Console")                     = 1 << 13;
+        PRIV_SYS_CONSOLE("Sys.Console");
     }
 }
 
 
+/// Admin always has all privileges. It can do everything except a few actions
+/// which are limited to the 'root@pam` superuser
 pub const ROLE_ADMIN: u64 = std::u64::MAX;
+
+/// NoAccess can be used to remove privileges from specific paths
 pub const ROLE_NO_ACCESS: u64 = 0;
 
 pub const ROLE_AUDIT: u64 =

src/config/cached_user_info.rs

@@ -64,10 +64,8 @@ impl CachedUserInfo {
                 return false;
             }
             if let Some(expire) = info.expire {
-                if expire > 0 {
-                    if expire <= now() {
-                        return false;
-                    }
+                if expire > 0 && expire <= now() {
+                    return false;
                 }
             }
             return true;
@@ -96,9 +94,7 @@ impl CachedUserInfo {
         }
         Ok(())
     }
-}
 
-impl CachedUserInfo {
     pub fn is_superuser(&self, userid: &Userid) -> bool {
         userid == "root@pam"
     }

src/config/datastore.rs

@@ -44,10 +44,6 @@ pub const DIR_NAME_SCHEMA: Schema = StringSchema::new("Directory name").schema()
             optional: true,
             schema: PRUNE_SCHEDULE_SCHEMA,
         },
-        "verify-schedule": {
-            optional: true,
-            schema: VERIFY_SCHEDULE_SCHEMA,
-        },
         "keep-last": {
             optional: true,
             schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -72,6 +68,10 @@ pub const DIR_NAME_SCHEMA: Schema = StringSchema::new("Directory name").schema()
             optional: true,
             schema: PRUNE_SCHEMA_KEEP_YEARLY,
         },
+        "verify-new": {
+            optional: true,
+            type: bool,
+        },
     }
 )]
 #[serde(rename_all="kebab-case")]
@@ -87,8 +87,6 @@ pub struct DataStoreConfig {
     #[serde(skip_serializing_if="Option::is_none")]
     pub prune_schedule: Option<String>,
     #[serde(skip_serializing_if="Option::is_none")]
-    pub verify_schedule: Option<String>,
-    #[serde(skip_serializing_if="Option::is_none")]
     pub keep_last: Option<u64>,
     #[serde(skip_serializing_if="Option::is_none")]
     pub keep_hourly: Option<u64>,
@@ -100,6 +98,9 @@ pub struct DataStoreConfig {
     pub keep_monthly: Option<u64>,
     #[serde(skip_serializing_if="Option::is_none")]
     pub keep_yearly: Option<u64>,
+    /// If enabled, all backups will be verified right after completion.
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub verify_new: Option<bool>,
 }
 
 fn init() -> SectionConfig {
@@ -157,12 +158,12 @@ pub fn complete_acl_path(_arg: &str, _param: &HashMap<String, String>) -> Vec<St
     let mut list = Vec::new();
 
     list.push(String::from("/"));
-    list.push(String::from("/storage"));
-    list.push(String::from("/storage/"));
+    list.push(String::from("/datastore"));
+    list.push(String::from("/datastore/"));
 
     if let Ok((data, _digest)) = config() {
         for id in data.sections.keys() {
-            list.push(format!("/storage/{}", id));
+            list.push(format!("/datastore/{}", id));
         }
     }
 

src/config/network.rs

@@ -289,8 +289,12 @@ impl Interface {
 
         if let Some(method6) = self.method6 {
             let mut skip_v6 = false; // avoid empty inet6 manual entry
-            if self.method.is_some() && method6 == NetworkConfigMethod::Manual {
-                if self.comments6.is_none() && self.options6.is_empty() { skip_v6 = true; }
+            if self.method.is_some()
+                && method6 == NetworkConfigMethod::Manual
+                && self.comments6.is_none()
+                && self.options6.is_empty()
+            {
+                skip_v6 = true;
             }
 
             if !skip_v6 {

src/config/network/helper.rs

@@ -10,7 +10,7 @@ use regex::Regex;
 
 use proxmox::*; // for IP macros
 
-pub static IPV4_REVERSE_MASK: &[&'static str] = &[
+pub static IPV4_REVERSE_MASK: &[&str] = &[
     "0.0.0.0",
     "128.0.0.0",
     "192.0.0.0",

src/config/verify.rs

@@ -0,0 +1,205 @@
+use anyhow::{Error};
+use lazy_static::lazy_static;
+use std::collections::HashMap;
+use serde::{Serialize, Deserialize};
+
+use proxmox::api::{
+    api,
+    schema::*,
+    section_config::{
+        SectionConfig,
+        SectionConfigData,
+        SectionConfigPlugin,
+    }
+};
+
+use proxmox::tools::{fs::replace_file, fs::CreateOptions};
+
+use crate::api2::types::*;
+
+lazy_static! {
+    static ref CONFIG: SectionConfig = init();
+}
+
+
+#[api(
+    properties: {
+        id: {
+            schema: JOB_ID_SCHEMA,
+        },
+        store: {
+            schema: DATASTORE_SCHEMA,
+        },
+        "ignore-verified": {
+            optional: true,
+            schema: IGNORE_VERIFIED_BACKUPS_SCHEMA,
+        },
+        "outdated-after": {
+            optional: true,
+            schema: VERIFICATION_OUTDATED_AFTER_SCHEMA,
+        },
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+        schedule: {
+            optional: true,
+            schema: VERIFICATION_SCHEDULE_SCHEMA,
+        },
+    }
+)]
+#[serde(rename_all="kebab-case")]
+#[derive(Serialize,Deserialize)]
+/// Verification Job
+pub struct VerificationJobConfig {
+    /// unique ID to address this job
+    pub id: String,
+    /// the datastore ID this verificaiton job affects
+    pub store: String,
+    #[serde(skip_serializing_if="Option::is_none")]
+    /// if not set to false, check the age of the last snapshot verification to filter
+    /// out recent ones, depending on 'outdated_after' configuration.
+    pub ignore_verified: Option<bool>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    /// Reverify snapshots after X days, never if 0. Ignored if 'ignore_verified' is false.
+    pub outdated_after: Option<i64>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub comment: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    /// when to schedule this job in calendar event notation
+    pub schedule: Option<String>,
+}
+
+
+#[api(
+    properties: {
+        id: {
+            schema: JOB_ID_SCHEMA,
+        },
+        store: {
+            schema: DATASTORE_SCHEMA,
+        },
+        "ignore-verified": {
+            optional: true,
+            schema: IGNORE_VERIFIED_BACKUPS_SCHEMA,
+        },
+        "outdated-after": {
+            optional: true,
+            schema: VERIFICATION_OUTDATED_AFTER_SCHEMA,
+        },
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+        schedule: {
+            optional: true,
+            schema: VERIFICATION_SCHEDULE_SCHEMA,
+        },
+        "next-run": {
+            description: "Estimated time of the next run (UNIX epoch).",
+            optional: true,
+            type: Integer,
+        },
+        "last-run-state": {
+            description: "Result of the last run.",
+            optional: true,
+            type: String,
+        },
+        "last-run-upid": {
+            description: "Task UPID of the last run.",
+            optional: true,
+            type: String,
+        },
+        "last-run-endtime": {
+            description: "Endtime of the last run.",
+            optional: true,
+            type: Integer,
+        },
+    }
+)]
+#[serde(rename_all="kebab-case")]
+#[derive(Serialize,Deserialize)]
+/// Status of Verification Job
+pub struct VerificationJobStatus {
+    /// unique ID to address this job
+    pub id: String,
+    /// the datastore ID this verificaiton job affects
+    pub store: String,
+    #[serde(skip_serializing_if="Option::is_none")]
+    /// if not set to false, check the age of the last snapshot verification to filter
+    /// out recent ones, depending on 'outdated_after' configuration.
+    pub ignore_verified: Option<bool>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    /// Reverify snapshots after X days, never if 0. Ignored if 'ignore_verified' is false.
+    pub outdated_after: Option<i64>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub comment: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    /// when to schedule this job in calendar event notation
+    pub schedule: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    /// The timestamp when this job runs the next time.
+    pub next_run: Option<i64>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    /// The state of the last scheduled run, if any
+    pub last_run_state: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    /// The task UPID of the last scheduled run, if any
+    pub last_run_upid: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    /// When the last run was finished, combined with UPID.starttime one can calculate the duration
+    pub last_run_endtime: Option<i64>,
+}
+
+
+fn init() -> SectionConfig {
+    let obj_schema = match VerificationJobConfig::API_SCHEMA {
+        Schema::Object(ref obj_schema) => obj_schema,
+        _ => unreachable!(),
+    };
+
+    let plugin = SectionConfigPlugin::new("verification".to_string(), Some(String::from("id")), obj_schema);
+    let mut config = SectionConfig::new(&JOB_ID_SCHEMA);
+    config.register_plugin(plugin);
+
+    config
+}
+
+pub const VERIFICATION_CFG_FILENAME: &str = "/etc/proxmox-backup/verification.cfg";
+pub const VERIFICATION_CFG_LOCKFILE: &str = "/etc/proxmox-backup/.verification.lck";
+
+pub fn config() -> Result<(SectionConfigData, [u8;32]), Error> {
+
+    let content = proxmox::tools::fs::file_read_optional_string(VERIFICATION_CFG_FILENAME)?;
+    let content = content.unwrap_or_else(String::new);
+
+    let digest = openssl::sha::sha256(content.as_bytes());
+    let data = CONFIG.parse(VERIFICATION_CFG_FILENAME, &content)?;
+    Ok((data, digest))
+}
+
+pub fn save_config(config: &SectionConfigData) -> Result<(), Error> {
+    let raw = CONFIG.write(VERIFICATION_CFG_FILENAME, &config)?;
+
+    let backup_user = crate::backup::backup_user()?;
+    let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640);
+    // set the correct owner/group/permissions while saving file
+    // owner(rw) = root, group(r)= backup
+
+    let options = CreateOptions::new()
+        .perm(mode)
+        .owner(nix::unistd::ROOT)
+        .group(backup_user.gid);
+
+    replace_file(VERIFICATION_CFG_FILENAME, raw.as_bytes(), options)?;
+
+    Ok(())
+}
+
+// shell completion helper
+pub fn complete_verification_job_id(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
+    match config() {
+        Ok((data, _digest)) => data.sections.iter().map(|(id, _)| id.to_string()).collect(),
+        Err(_) => return vec![],
+    }
+}

src/lib.rs

@@ -1,3 +1,10 @@
+//! See the different modules for documentation on their usage.
+//!
+//! The [backup](backup/index.html) module contains some detailed information
+//! on the inner workings of the backup server regarding data storage.
+
+pub mod task;
+
 #[macro_use]
 pub mod buildcfg;
 

src/pxar/create.rs

@@ -12,7 +12,7 @@ use nix::errno::Errno;
 use nix::fcntl::OFlag;
 use nix::sys::stat::{FileStat, Mode};
 
-use pathpatterns::{MatchEntry, MatchList, MatchType, PatternFlag};
+use pathpatterns::{MatchEntry, MatchFlag, MatchList, MatchType, PatternFlag};
 use pxar::Metadata;
 use pxar::encoder::LinkOffset;
 
@@ -235,7 +235,7 @@ impl<'a, 'b> Archiver<'a, 'b> {
                 continue;
             }
 
-            (self.callback)(Path::new(OsStr::from_bytes(file_name)))?;
+            (self.callback)(&file_entry.path)?;
             self.path = file_entry.path;
             self.add_entry(encoder, dir_fd, &file_entry.name, &file_entry.stat)
                 .map_err(|err| self.wrap_err(err))?;
@@ -291,59 +291,68 @@ impl<'a, 'b> Archiver<'a, 'b> {
     }
 
     fn read_pxar_excludes(&mut self, parent: RawFd) -> Result<(), Error> {
-        let fd = self.open_file(parent, c_str!(".pxarexclude"), OFlag::O_RDONLY, false)?;
+        let fd = match self.open_file(parent, c_str!(".pxarexclude"), OFlag::O_RDONLY, false)? {
+            Some(fd) => fd,
+            None => return Ok(()),
+        };
 
         let old_pattern_count = self.patterns.len();
 
         let path_bytes = self.path.as_os_str().as_bytes();
 
-        if let Some(fd) = fd {
-            let file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) };
+        let file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) };
 
-            use io::BufRead;
-            for line in io::BufReader::new(file).split(b'\n') {
-                let line = match line {
-                    Ok(line) => line,
-                    Err(err) => {
-                        let _ = writeln!(
-                            self.errors,
-                            "ignoring .pxarexclude after read error in {:?}: {}",
-                            self.path,
-                            err,
-                        );
-                        self.patterns.truncate(old_pattern_count);
-                        return Ok(());
-                    }
-                };
-
-                let line = crate::tools::strip_ascii_whitespace(&line);
-
-                if line.is_empty() || line[0] == b'#' {
-                    continue;
+        use io::BufRead;
+        for line in io::BufReader::new(file).split(b'\n') {
+            let line = match line {
+                Ok(line) => line,
+                Err(err) => {
+                    let _ = writeln!(
+                        self.errors,
+                        "ignoring .pxarexclude after read error in {:?}: {}",
+                        self.path,
+                        err,
+                    );
+                    self.patterns.truncate(old_pattern_count);
+                    return Ok(());
                 }
+            };
 
-                let mut buf;
-                let (line, mode) = if line[0] == b'/' {
-                    buf = Vec::with_capacity(path_bytes.len() + 1 + line.len());
-                    buf.extend(path_bytes);
-                    buf.extend(line);
-                    (&buf[..], MatchType::Exclude)
-                } else if line.starts_with(b"!/") {
-                    // inverted case with absolute path
-                    buf = Vec::with_capacity(path_bytes.len() + line.len());
-                    buf.extend(path_bytes);
-                    buf.extend(&line[1..]); // without the '!'
-                    (&buf[..], MatchType::Include)
-                } else {
-                    (line, MatchType::Exclude)
-                };
+            let line = crate::tools::strip_ascii_whitespace(&line);
 
-                match MatchEntry::parse_pattern(line, PatternFlag::PATH_NAME, mode) {
-                    Ok(pattern) => self.patterns.push(pattern),
-                    Err(err) => {
-                        let _ = writeln!(self.errors, "bad pattern in {:?}: {}", self.path, err);
+            if line.is_empty() || line[0] == b'#' {
+                continue;
+            }
+
+            let mut buf;
+            let (line, mode, anchored) = if line[0] == b'/' {
+                buf = Vec::with_capacity(path_bytes.len() + 1 + line.len());
+                buf.extend(path_bytes);
+                buf.extend(line);
+                (&buf[..], MatchType::Exclude, true)
+            } else if line.starts_with(b"!/") {
+                // inverted case with absolute path
+                buf = Vec::with_capacity(path_bytes.len() + line.len());
+                buf.extend(path_bytes);
+                buf.extend(&line[1..]); // without the '!'
+                (&buf[..], MatchType::Include, true)
+            } else if line.starts_with(b"!") {
+                (&line[1..], MatchType::Include, false)
+            } else {
+                (line, MatchType::Exclude, false)
+            };
+
+            match MatchEntry::parse_pattern(line, PatternFlag::PATH_NAME, mode) {
+                Ok(pattern) => {
+                    if anchored {
+                        self.patterns.push(pattern.add_flags(MatchFlag::ANCHORED));
+                    } else {
+                        self.patterns.push(pattern);
                     }
                 }
+                Err(err) => {
+                    let _ = writeln!(self.errors, "bad pattern in {:?}: {}", self.path, err);
+                }
             }
         }
 
@@ -997,7 +1006,7 @@ fn process_acl(
 /// Since we are generating an *exclude* list, we need to invert this, so includes get a `'!'`
 /// prefix.
 fn generate_pxar_excludes_cli(patterns: &[MatchEntry]) -> Vec<u8> {
-    use pathpatterns::{MatchFlag, MatchPattern};
+    use pathpatterns::MatchPattern;
 
     let mut content = Vec::new();
 

src/pxar/fuse.rs

@@ -266,10 +266,8 @@ impl SessionImpl {
     ) {
         let final_result = match err.downcast::<io::Error>() {
             Ok(err) => {
-                if err.kind() == io::ErrorKind::Other {
-                    if self.verbose {
-                        eprintln!("an IO error occurred: {}", err);
-                    }
+                if err.kind() == io::ErrorKind::Other && self.verbose {
+                    eprintln!("an IO error occurred: {}", err);
                 }
 
                 // fail the request

src/rrd/rrd.rs

@@ -272,7 +272,7 @@ impl RRD {
             t += reso; index = (index + 1) % RRD_DATA_ENTRIES;
         }
 
-        (start, reso, list.into())
+        (start, reso, list)
     }
 
     pub fn from_raw(mut raw: &[u8]) -> Result<Self, std::io::Error> {
@@ -289,7 +289,7 @@ impl RRD {
         }
 
         if rrd.magic != PROXMOX_RRD_MAGIC_1_0 {
-            let msg = format!("wrong magic number");
+            let msg = "wrong magic number".to_string();
             return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
         }
 

src/server.rs

@@ -30,3 +30,10 @@ pub mod formatter;
 #[macro_use]
 pub mod rest;
 
+pub mod jobstate;
+
+mod verify_job;
+pub use verify_job::*;
+
+mod email_notifications;
+pub use email_notifications::*;

src/server/config.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 use std::path::PathBuf;
 use std::time::SystemTime;
 use std::fs::metadata;
-use std::sync::RwLock;
+use std::sync::{Mutex, RwLock};
 
 use anyhow::{bail, Error, format_err};
 use hyper::Method;
@@ -10,6 +10,9 @@ use handlebars::Handlebars;
 use serde::Serialize;
 
 use proxmox::api::{ApiMethod, Router, RpcEnvironmentType};
+use proxmox::tools::fs::{create_path, CreateOptions};
+
+use crate::tools::{FileLogger, FileLogOptions};
 
 pub struct ApiConfig {
     basedir: PathBuf,
@@ -18,6 +21,7 @@ pub struct ApiConfig {
     env_type: RpcEnvironmentType,
     templates: RwLock<Handlebars<'static>>,
     template_files: RwLock<HashMap<String, (SystemTime, PathBuf)>>,
+    request_log: Option<Mutex<FileLogger>>,
 }
 
 impl ApiConfig {
@@ -30,6 +34,7 @@ impl ApiConfig {
             env_type,
             templates: RwLock::new(Handlebars::new()),
             template_files: RwLock::new(HashMap::new()),
+            request_log: None,
         })
     }
 
@@ -118,4 +123,30 @@ impl ApiConfig {
             templates.render(name, data).map_err(|err| format_err!("{}", err))
         }
     }
+
+    pub fn enable_file_log<P>(&mut self, path: P) -> Result<(), Error>
+    where
+        P: Into<PathBuf>
+    {
+        let path: PathBuf = path.into();
+        if let Some(base) = path.parent() {
+            if !base.exists() {
+                let backup_user = crate::backup::backup_user()?;
+                let opts = CreateOptions::new().owner(backup_user.uid).group(backup_user.gid);
+                create_path(base, None, Some(opts)).map_err(|err| format_err!("{}", err))?;
+            }
+        }
+
+        let logger_options = FileLogOptions {
+            append: true,
+            owned_by_backup: true,
+            ..Default::default()
+        };
+        self.request_log = Some(Mutex::new(FileLogger::new(&path, logger_options)?));
+
+        Ok(())
+    }
+    pub fn get_file_log(&self) -> Option<&Mutex<FileLogger>> {
+        self.request_log.as_ref()
+    }
 }

src/server/email_notifications.rs

@@ -0,0 +1,250 @@
+use anyhow::Error;
+use serde_json::json;
+
+use handlebars::{Handlebars, Helper, Context, RenderError, RenderContext, Output, HelperResult};
+
+use proxmox::tools::email::sendmail;
+
+use crate::{
+    config::verify::VerificationJobConfig,
+    api2::types::{
+        Userid,
+        GarbageCollectionStatus,
+    },
+    tools::format::HumanByte,
+};
+
+const GC_OK_TEMPLATE: &str = r###"
+
+Datastore:           {{datastore}}
+Task ID:             {{status.upid}}
+Index file count:    {{status.index-file-count}}
+
+Removed garbage:     {{human-bytes status.removed-bytes}}
+Removed chunks:      {{status.removed-chunks}}
+Remove bad files:    {{status.removed-bad}}
+
+Pending removals:    {{human-bytes status.pending-bytes}} (in {{status.pending-chunks}} chunks)
+
+Original Data usage: {{human-bytes status.index-data-bytes}}
+On Disk usage:       {{human-bytes status.disk-bytes}} ({{relative-percentage status.disk-bytes status.index-data-bytes}})
+On Disk chunks:      {{status.disk-chunks}}
+
+Garbage collection successful.
+
+"###;
+
+
+const GC_ERR_TEMPLATE: &str = r###"
+
+Datastore: {{datastore}}
+
+Garbage collection failed: {{error}}
+
+"###;
+
+const VERIFY_OK_TEMPLATE: &str = r###"
+
+Job ID:    {{job.id}}
+Datastore: {{job.store}}
+
+Verification successful.
+
+"###;
+
+const VERIFY_ERR_TEMPLATE: &str = r###"
+
+Job ID:    {{job.id}}
+Datastore: {{job.store}}
+
+Verification failed on these snapshots:
+
+{{#each errors}}
+  {{this}}
+{{/each}}
+
+"###;
+
+lazy_static::lazy_static!{
+
+    static ref HANDLEBARS: Handlebars<'static> = {
+        let mut hb = Handlebars::new();
+
+        hb.set_strict_mode(true);
+
+        hb.register_helper("human-bytes", Box::new(handlebars_humam_bytes_helper));
+        hb.register_helper("relative-percentage", Box::new(handlebars_relative_percentage_helper));
+
+        hb.register_template_string("gc_ok_template", GC_OK_TEMPLATE).unwrap();
+        hb.register_template_string("gc_err_template", GC_ERR_TEMPLATE).unwrap();
+
+        hb.register_template_string("verify_ok_template", VERIFY_OK_TEMPLATE).unwrap();
+        hb.register_template_string("verify_err_template", VERIFY_ERR_TEMPLATE).unwrap();
+
+        hb
+    };
+}
+
+fn send_job_status_mail(
+    email: &str,
+    subject: &str,
+    text: &str,
+) -> Result<(), Error> {
+
+    // Note: OX has serious problems displaying text mails,
+    // so we include html as well
+    let html = format!("<html><body><pre>\n{}\n<pre>", text);
+
+    let nodename = proxmox::tools::nodename();
+
+    let author = format!("Proxmox Backup Server - {}", nodename);
+
+    sendmail(
+        &[email],
+        &subject,
+        Some(&text),
+        Some(&html),
+        None,
+        Some(&author),
+    )?;
+
+    Ok(())
+}
+
+pub fn send_gc_status(
+    email: &str,
+    datastore: &str,
+    status: &GarbageCollectionStatus,
+    result: &Result<(), Error>,
+) -> Result<(), Error> {
+
+    let text = match result {
+        Ok(()) => {
+            let data = json!({
+                "status": status,
+                "datastore": datastore,
+            });
+            HANDLEBARS.render("gc_ok_template", &data)?
+        }
+        Err(err) => {
+            let data = json!({
+                "error": err.to_string(),
+                "datastore": datastore,
+            });
+            HANDLEBARS.render("gc_err_template", &data)?
+        }
+    };
+
+    let subject = match result {
+        Ok(()) => format!(
+            "Garbage Collect Datastore '{}' successful",
+            datastore,
+        ),
+        Err(_) => format!(
+            "Garbage Collect Datastore '{}' failed",
+            datastore,
+        ),
+    };
+
+    send_job_status_mail(email, &subject, &text)?;
+
+    Ok(())
+}
+
+pub fn send_verify_status(
+    email: &str,
+    job: VerificationJobConfig,
+    result: &Result<Vec<String>, Error>,
+) -> Result<(), Error> {
+
+
+    let text = match result {
+        Ok(errors) if errors.is_empty() => {
+            let data = json!({ "job": job });
+            HANDLEBARS.render("verify_ok_template", &data)?
+        }
+        Ok(errors) => {
+            let data = json!({ "job": job, "errors": errors });
+            HANDLEBARS.render("verify_err_template", &data)?
+        }
+        Err(_) => {
+            // aboreted job - do not send any email
+            return Ok(());
+        }
+    };
+
+    let subject = match result {
+        Ok(errors) if errors.is_empty() => format!(
+            "Verify Datastore '{}' successful",
+            job.store,
+        ),
+        _ => format!(
+            "Verify Datastore '{}' failed",
+            job.store,
+        ),
+    };
+
+    send_job_status_mail(email, &subject, &text)?;
+
+    Ok(())
+}
+
+/// Lookup users email address
+///
+/// For "backup@pam", this returns the address from "root@pam".
+pub fn lookup_user_email(userid: &Userid) -> Option<String> {
+
+    use crate::config::user::{self, User};
+
+    if userid == Userid::backup_userid() {
+        return lookup_user_email(Userid::root_userid());
+    }
+
+    if let Ok(user_config) = user::cached_config() {
+        if let Ok(user) = user_config.lookup::<User>("user", userid.as_str()) {
+            return user.email.clone();
+        }
+    }
+
+    None
+}
+
+// Handlerbar helper functions
+
+fn handlebars_humam_bytes_helper(
+    h: &Helper,
+    _: &Handlebars,
+    _: &Context,
+    _rc: &mut RenderContext,
+    out: &mut dyn Output
+) -> HelperResult {
+    let param = h.param(0).map(|v| v.value().as_u64())
+        .flatten()
+        .ok_or(RenderError::new("human-bytes: param not found"))?;
+
+    out.write(&HumanByte::from(param).to_string())?;
+
+    Ok(())
+}
+
+fn handlebars_relative_percentage_helper(
+    h: &Helper,
+    _: &Handlebars,
+    _: &Context,
+    _rc: &mut RenderContext,
+    out: &mut dyn Output
+) -> HelperResult {
+    let param0 = h.param(0).map(|v| v.value().as_f64())
+        .flatten()
+        .ok_or(RenderError::new("relative-percentage: param0 not found"))?;
+    let param1 = h.param(1).map(|v| v.value().as_f64())
+        .flatten()
+        .ok_or(RenderError::new("relative-percentage: param1 not found"))?;
+
+    if param1 == 0.0 {
+        out.write("-")?;
+    } else {
+        out.write(&format!("{:.2}%", (param0*100.0)/param1))?;
+    }
+    Ok(())
+}

src/server/environment.rs

@@ -7,6 +7,7 @@ pub struct RestEnvironment {
     env_type: RpcEnvironmentType,
     result_attributes: Value,
     user: Option<String>,
+    client_ip: Option<std::net::SocketAddr>,
 }
 
 impl RestEnvironment {
@@ -14,6 +15,7 @@ impl RestEnvironment {
         Self {
             result_attributes: json!({}),
             user: None,
+            client_ip: None,
             env_type,
         }
     }
@@ -40,4 +42,12 @@ impl RpcEnvironment for RestEnvironment {
     fn get_user(&self) -> Option<String> {
         self.user.clone()
     }
+
+    fn set_client_ip(&mut self, client_ip: Option<std::net::SocketAddr>) {
+        self.client_ip = client_ip;
+    }
+
+    fn get_client_ip(&self) -> Option<std::net::SocketAddr> {
+        self.client_ip.clone()
+    }
 }

src/server/jobstate.rs

@@ -15,7 +15,7 @@
 //! ```no_run
 //! # use anyhow::{bail, Error};
 //! # use proxmox_backup::server::TaskState;
-//! # use proxmox_backup::config::jobstate::*;
+//! # use proxmox_backup::server::jobstate::*;
 //! # fn some_code() -> TaskState { TaskState::OK { endtime: 0 } }
 //! # fn code() -> Result<(), Error> {
 //! // locks the correct file under /var/lib

src/server/rest.rs

@@ -3,24 +3,39 @@ use std::future::Future;
 use std::hash::BuildHasher;
 use std::path::{Path, PathBuf};
 use std::pin::Pin;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use std::task::{Context, Poll};
 
 use anyhow::{bail, format_err, Error};
 use futures::future::{self, FutureExt, TryFutureExt};
 use futures::stream::TryStreamExt;
-use hyper::header;
+use hyper::header::{self, HeaderMap};
+use hyper::body::HttpBody;
 use hyper::http::request::Parts;
 use hyper::{Body, Request, Response, StatusCode};
+use lazy_static::lazy_static;
 use serde_json::{json, Value};
 use tokio::fs::File;
 use tokio::time::Instant;
 use url::form_urlencoded;
+use regex::Regex;
 
 use proxmox::http_err;
-use proxmox::api::{ApiHandler, ApiMethod, HttpError};
-use proxmox::api::{RpcEnvironment, RpcEnvironmentType, check_api_permission};
-use proxmox::api::schema::{ObjectSchema, parse_simple_value, verify_json_object, parse_parameter_strings};
+use proxmox::api::{
+    ApiHandler,
+    ApiMethod,
+    HttpError,
+    Permission,
+    RpcEnvironment,
+    RpcEnvironmentType,
+    check_api_permission,
+};
+use proxmox::api::schema::{
+    ObjectSchema,
+    parse_parameter_strings,
+    parse_simple_value,
+    verify_json_object,
+};
 
 use super::environment::RestEnvironment;
 use super::formatter::*;
@@ -29,6 +44,7 @@ use super::ApiConfig;
 use crate::auth_helpers::*;
 use crate::api2::types::Userid;
 use crate::tools;
+use crate::tools::FileLogger;
 use crate::tools::ticket::Ticket;
 use crate::config::cached_user_info::CachedUserInfo;
 
@@ -38,6 +54,8 @@ pub struct RestServer {
     pub api_config: Arc<ApiConfig>,
 }
 
+const MAX_URI_QUERY_LENGTH: usize = 3072;
+
 impl RestServer {
 
     pub fn new(api_config: ApiConfig) -> Self {
@@ -93,14 +111,20 @@ pub struct ApiService {
 }
 
 fn log_response(
+    logfile: Option<&Mutex<FileLogger>>,
     peer: &std::net::SocketAddr,
     method: hyper::Method,
-    path: &str,
+    path_query: &str,
     resp: &Response<Body>,
+    user_agent: Option<String>,
 ) {
 
     if resp.extensions().get::<NoLogExtension>().is_some() { return; };
 
+    // we also log URL-to-long requests, so avoid message bigger than PIPE_BUF (4k on Linux)
+    // to profit from atomicty guarantees for O_APPEND opened logfiles
+    let path = &path_query[..MAX_URI_QUERY_LENGTH.min(path_query.len())];
+
     let status = resp.status();
 
     if !(status.is_success() || status.is_informational()) {
@@ -113,6 +137,51 @@ fn log_response(
 
         log::error!("{} {}: {} {}: [client {}] {}", method.as_str(), path, status.as_str(), reason, peer, message);
     }
+    if let Some(logfile) = logfile {
+        let user = match resp.extensions().get::<Userid>() {
+            Some(userid) => userid.as_str(),
+            None => "-",
+        };
+        let now = proxmox::tools::time::epoch_i64();
+        // time format which apache/nginx use (by default), copied from pve-http-server
+        let datetime = proxmox::tools::time::strftime_local("%d/%m/%Y:%H:%M:%S %z", now)
+            .unwrap_or("-".into());
+
+        logfile
+            .lock()
+            .unwrap()
+            .log(format!(
+                "{} - {} [{}] \"{} {}\" {} {} {}",
+                peer.ip(),
+                user,
+                datetime,
+                method.as_str(),
+                path,
+                status.as_str(),
+                resp.body().size_hint().lower(),
+                user_agent.unwrap_or("-".into()),
+            ));
+    }
+}
+
+fn get_proxied_peer(headers: &HeaderMap) -> Option<std::net::SocketAddr> {
+    lazy_static! {
+        static ref RE: Regex = Regex::new(r#"for="([^"]+)""#).unwrap();
+    }
+    let forwarded = headers.get(header::FORWARDED)?.to_str().ok()?;
+    let capture = RE.captures(&forwarded)?;
+    let rhost = capture.get(1)?.as_str();
+
+    rhost.parse().ok()
+}
+
+fn get_user_agent(headers: &HeaderMap) -> Option<String> {
+    let agent = headers.get(header::USER_AGENT)?.to_str();
+    agent.map(|s| {
+        let mut s = s.to_owned();
+        s.truncate(128);
+        s
+    }).ok()
 }
 
 impl tower_service::Service<Request<Body>> for ApiService {
@@ -125,31 +194,31 @@ impl tower_service::Service<Request<Body>> for ApiService {
     }
 
     fn call(&mut self, req: Request<Body>) -> Self::Future {
-        let path = req.uri().path().to_owned();
+        let path = req.uri().path_and_query().unwrap().as_str().to_owned();
         let method = req.method().clone();
+        let user_agent = get_user_agent(req.headers());
 
-        let peer = self.peer;
-        handle_request(self.api_config.clone(), req)
-            .map(move |result| match result {
-                Ok(res) => {
-                    log_response(&peer, method, &path, &res);
-                    Ok::<_, Self::Error>(res)
-                }
+        let config = Arc::clone(&self.api_config);
+        let peer = match get_proxied_peer(req.headers()) {
+            Some(proxied_peer) => proxied_peer,
+            None => self.peer,
+        };
+        async move {
+            let response = match handle_request(Arc::clone(&config), req, &peer).await {
+                Ok(response) => response,
                 Err(err) => {
-                    if let Some(apierr) = err.downcast_ref::<HttpError>() {
-                        let mut resp = Response::new(Body::from(apierr.message.clone()));
-                        *resp.status_mut() = apierr.code;
-                        log_response(&peer, method, &path, &resp);
-                        Ok(resp)
-                    } else {
-                        let mut resp = Response::new(Body::from(err.to_string()));
-                        *resp.status_mut() = StatusCode::BAD_REQUEST;
-                        log_response(&peer, method, &path, &resp);
-                        Ok(resp)
-                    }
+                    let (err, code) = match err.downcast_ref::<HttpError>() {
+                        Some(apierr) => (apierr.message.clone(), apierr.code),
+                        _ => (err.to_string(), StatusCode::BAD_REQUEST),
+                    };
+                    Response::builder().status(code).body(err.into())?
                 }
-            })
-            .boxed()
+            };
+            let logger = config.get_file_log();
+            log_response(logger, &peer, method, &path, &response, user_agent);
+            Ok(response)
+        }
+        .boxed()
     }
 }
 
@@ -239,6 +308,7 @@ async fn proxy_protected_request(
     info: &'static ApiMethod,
     mut parts: Parts,
     req_body: Body,
+    peer: &std::net::SocketAddr,
 ) -> Result<Response<Body>, Error> {
 
     let mut uri_parts = parts.uri.clone().into_parts();
@@ -249,7 +319,10 @@ async fn proxy_protected_request(
 
     parts.uri = new_uri;
 
-    let request = Request::from_parts(parts, req_body);
+    let mut request = Request::from_parts(parts, req_body);
+    request
+        .headers_mut()
+        .insert(header::FORWARDED, format!("for=\"{}\";", peer).parse().unwrap());
 
     let reload_timezone = info.reload_timezone;
 
@@ -315,16 +388,16 @@ pub async fn handle_api_request<Env: RpcEnvironment, S: 'static + BuildHasher +
 
 fn get_index(
     userid: Option<Userid>,
-    token: Option<String>,
+    csrf_token: Option<String>,
     language: Option<String>,
     api: &Arc<ApiConfig>,
     parts: Parts,
 ) ->  Response<Body> {
 
     let nodename = proxmox::tools::nodename();
-    let userid = userid.as_ref().map(|u| u.as_str()).unwrap_or("");
+    let user = userid.as_ref().map(|u| u.as_str()).unwrap_or("");
 
-    let token = token.unwrap_or_else(|| String::from(""));
+    let csrf_token = csrf_token.unwrap_or_else(|| String::from(""));
 
     let mut debug = false;
     let mut template_file = "index";
@@ -348,27 +421,30 @@ fn get_index(
 
     let data = json!({
         "NodeName": nodename,
-        "UserName": userid,
-        "CSRFPreventionToken": token,
+        "UserName": user,
+        "CSRFPreventionToken": csrf_token,
         "language": lang,
         "debug": debug,
     });
 
-    let mut ct = "text/html";
-
-    let index = match api.render_template(template_file, &data) {
-        Ok(index) => index,
+    let (ct, index) = match api.render_template(template_file, &data) {
+        Ok(index) => ("text/html", index),
         Err(err) => {
-            ct = "text/plain";
-            format!("Error rendering template: {}", err)
+            ("text/plain", format!("Error rendering template: {}", err))
         }
     };
 
-    Response::builder()
+    let mut resp = Response::builder()
         .status(StatusCode::OK)
         .header(header::CONTENT_TYPE, ct)
         .body(index.into())
-        .unwrap()
+        .unwrap();
+
+    if let Some(userid) = userid {
+        resp.extensions_mut().insert(userid);
+    }
+
+    resp
 }
 
 fn extension_to_content_type(filename: &Path) -> (&'static str, bool) {
@@ -466,18 +542,18 @@ fn extract_auth_data(headers: &http::HeaderMap) -> (Option<String>, Option<Strin
         }
     }
 
-    let token = match headers.get("CSRFPreventionToken").map(|v| v.to_str()) {
+    let csrf_token = match headers.get("CSRFPreventionToken").map(|v| v.to_str()) {
         Some(Ok(v)) => Some(v.to_owned()),
         _ => None,
     };
 
-    (ticket, token, language)
+    (ticket, csrf_token, language)
 }
 
 fn check_auth(
     method: &hyper::Method,
     ticket: &Option<String>,
-    token: &Option<String>,
+    csrf_token: &Option<String>,
     user_info: &CachedUserInfo,
 ) -> Result<Userid, Error> {
     let ticket_lifetime = tools::ticket::TICKET_LIFETIME;
@@ -491,9 +567,8 @@ fn check_auth(
     }
 
     if method != hyper::Method::GET {
-        if let Some(token) = token {
-            println!("CSRF prevention token: {:?}", token);
-            verify_csrf_prevention_token(csrf_secret(), &userid, &token, -300, ticket_lifetime)?;
+        if let Some(csrf_token) = csrf_token {
+            verify_csrf_prevention_token(csrf_secret(), &userid, &csrf_token, -300, ticket_lifetime)?;
         } else {
             bail!("missing CSRF prevention token");
         }
@@ -502,21 +577,31 @@ fn check_auth(
     Ok(userid)
 }
 
-pub async fn handle_request(api: Arc<ApiConfig>, req: Request<Body>) -> Result<Response<Body>, Error> {
+async fn handle_request(
+    api: Arc<ApiConfig>,
+    req: Request<Body>,
+    peer: &std::net::SocketAddr,
+) -> Result<Response<Body>, Error> {
 
     let (parts, body) = req.into_parts();
-
     let method = parts.method.clone();
     let (path, components) = tools::normalize_uri_path(parts.uri.path())?;
 
     let comp_len = components.len();
 
-    //println!("REQUEST {} {}", method, path);
-    //println!("COMPO {:?}", components);
+    let query = parts.uri.query().unwrap_or_default();
+    if path.len() + query.len() > MAX_URI_QUERY_LENGTH {
+        return Ok(Response::builder()
+            .status(StatusCode::URI_TOO_LONG)
+            .body("".into())
+            .unwrap());
+    }
 
     let env_type = api.env_type();
     let mut rpcenv = RestEnvironment::new(env_type);
 
+    rpcenv.set_client_ip(Some(*peer));
+
     let user_info = CachedUserInfo::new()?;
 
     let delay_unauth_time = std::time::Instant::now() + std::time::Duration::from_millis(3000);
@@ -535,15 +620,18 @@ pub async fn handle_request(api: Arc<ApiConfig>, req: Request<Body>) -> Result<R
             };
 
             let mut uri_param = HashMap::new();
+            let api_method = api.find_method(&components[2..], method.clone(), &mut uri_param);
 
-            if comp_len == 4 && components[2] == "access" && (
-                (components[3] == "ticket" && method ==  hyper::Method::POST) ||
-                (components[3] == "domains" && method ==  hyper::Method::GET)
-            ) {
-                // explicitly allow those calls without auth
-            } else {
-                let (ticket, token, _) = extract_auth_data(&parts.headers);
-                match check_auth(&method, &ticket, &token, &user_info) {
+            let mut auth_required = true;
+            if let Some(api_method) = api_method {
+                if let Permission::World = *api_method.access.permission {
+                    auth_required = false; // no auth for endpoints with World permission
+                }
+            }
+
+            if auth_required {
+                let (ticket, csrf_token, _) = extract_auth_data(&parts.headers);
+                match check_auth(&method, &ticket, &csrf_token, &user_info) {
                     Ok(userid) => rpcenv.set_user(Some(userid.to_string())),
                     Err(err) => {
                         // always delay unauthorized calls by 3 seconds (from start of request)
@@ -554,7 +642,7 @@ pub async fn handle_request(api: Arc<ApiConfig>, req: Request<Body>) -> Result<R
                 }
             }
 
-            match api.find_method(&components[2..], method, &mut uri_param) {
+            match api_method {
                 None => {
                     let err = http_err!(NOT_FOUND, "Path '{}' not found.", path);
                     return Ok((formatter.format_error)(err));
@@ -568,15 +656,22 @@ pub async fn handle_request(api: Arc<ApiConfig>, req: Request<Body>) -> Result<R
                     }
 
                     let result = if api_method.protected && env_type == RpcEnvironmentType::PUBLIC {
-                        proxy_protected_request(api_method, parts, body).await
+                        proxy_protected_request(api_method, parts, body, peer).await
                     } else {
                         handle_api_request(rpcenv, api_method, formatter, parts, body, uri_param).await
                     };
 
-                    if let Err(err) = result {
-                        return Ok((formatter.format_error)(err));
+                    let mut response = match result {
+                        Ok(resp) => resp,
+                        Err(err) => (formatter.format_error)(err),
+                    };
+
+                    if let Some(user) = user {
+                        let userid: Userid = user.parse()?;
+                        response.extensions_mut().insert(userid);
                     }
-                    return result;
+
+                    return Ok(response);
                 }
             }
 
@@ -589,12 +684,12 @@ pub async fn handle_request(api: Arc<ApiConfig>, req: Request<Body>) -> Result<R
         }
 
         if comp_len == 0 {
-            let (ticket, token, language) = extract_auth_data(&parts.headers);
+            let (ticket, csrf_token, language) = extract_auth_data(&parts.headers);
             if ticket != None {
-                match check_auth(&method, &ticket, &token, &user_info) {
+                match check_auth(&method, &ticket, &csrf_token, &user_info) {
                     Ok(userid) => {
-                        let new_token = assemble_csrf_prevention_token(csrf_secret(), &userid);
-                        return Ok(get_index(Some(userid), Some(new_token), language, &api, parts));
+                        let new_csrf_token = assemble_csrf_prevention_token(csrf_secret(), &userid);
+                        return Ok(get_index(Some(userid), Some(new_csrf_token), language, &api, parts));
                     }
                     _ => {
                         tokio::time::delay_until(Instant::from_std(delay_unauth_time)).await;

src/server/upid.rs

@@ -75,11 +75,6 @@ impl UPID {
         if worker_type.contains(bad) {
             bail!("illegal characters in worker type '{}'", worker_type);
         }
-        if let Some(ref worker_id) = worker_id {
-            if worker_id.contains(bad) {
-                bail!("illegal characters in worker id '{}'", worker_id);
-            }
-        }
 
         static WORKER_TASK_NEXT_ID: AtomicUsize = AtomicUsize::new(0);
 
@@ -112,13 +107,21 @@ impl std::str::FromStr for UPID {
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         if let Some(cap) = PROXMOX_UPID_REGEX.captures(s) {
+
+            let worker_id = if cap["wid"].is_empty() {
+                None
+            } else {
+                let wid = crate::tools::systemd::unescape_unit(&cap["wid"])?;
+                Some(wid)
+            };
+
             Ok(UPID {
                 pid: i32::from_str_radix(&cap["pid"], 16).unwrap(),
                 pstart: u64::from_str_radix(&cap["pstart"], 16).unwrap(),
                 starttime: i64::from_str_radix(&cap["starttime"], 16).unwrap(),
                 task_id: usize::from_str_radix(&cap["task_id"], 16).unwrap(),
                 worker_type: cap["wtype"].to_string(),
-                worker_id: if cap["wid"].is_empty() { None } else { Some(cap["wid"].to_string()) },
+                worker_id,
                 userid: cap["userid"].parse()?,
                 node: cap["node"].to_string(),
             })
@@ -133,7 +136,11 @@ impl std::fmt::Display for UPID {
 
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
 
-        let wid = if let Some(ref id) = self.worker_id { id } else { "" };
+        let wid = if let Some(ref id) = self.worker_id {
+            crate::tools::systemd::escape_unit(id, false)
+        } else {
+            String::new()
+        };
 
         // Note: pstart can be > 32bit if uptime > 497 days, so this can result in
         // more that 8 characters for pstart

src/server/verify_job.rs

@@ -0,0 +1,102 @@
+use anyhow::{format_err, Error};
+
+use crate::{
+    server::WorkerTask,
+    api2::types::*,
+    server::jobstate::Job,
+    config::verify::VerificationJobConfig,
+    backup::{
+        DataStore,
+        BackupInfo,
+        verify_all_backups,
+    },
+    task_log,
+};
+
+/// Runs a verification job.
+pub fn do_verification_job(
+    mut job: Job,
+    verification_job: VerificationJobConfig,
+    userid: &Userid,
+    schedule: Option<String>,
+) -> Result<String, Error> {
+
+    let datastore = DataStore::lookup_datastore(&verification_job.store)?;
+
+    let datastore2 = datastore.clone();
+
+    let outdated_after = verification_job.outdated_after.clone();
+    let ignore_verified_snapshots = verification_job.ignore_verified.unwrap_or(true);
+
+    let filter = move |backup_info: &BackupInfo| {
+        if !ignore_verified_snapshots {
+            return true;
+        }
+        let manifest = match datastore2.load_manifest(&backup_info.backup_dir) {
+            Ok((manifest, _)) => manifest,
+            Err(_) => return true, // include, so task picks this up as error
+        };
+
+        let raw_verify_state = manifest.unprotected["verify_state"].clone();
+        match serde_json::from_value::<SnapshotVerifyState>(raw_verify_state) {
+            Err(_) => return true, // no last verification, always include
+            Ok(last_verify) => {
+                match outdated_after {
+                    None => false, // never re-verify if ignored and no max age
+                    Some(max_age) => {
+                        let now = proxmox::tools::time::epoch_i64();
+                        let days_since_last_verify = (now - last_verify.upid.starttime) / 86400;
+
+                        days_since_last_verify > max_age
+                    }
+                }
+            }
+        }
+    };
+
+    let email = crate::server::lookup_user_email(userid);
+
+    let job_id = job.jobname().to_string();
+    let worker_type = job.jobtype().to_string();
+    let upid_str = WorkerTask::new_thread(
+        &worker_type,
+        Some(job.jobname().to_string()),
+        userid.clone(),
+        false,
+        move |worker| {
+            job.start(&worker.upid().to_string())?;
+
+            task_log!(worker,"Starting datastore verify job '{}'", job_id);
+            if let Some(event_str) = schedule {
+                task_log!(worker,"task triggered by schedule '{}'", event_str);
+            }
+
+            let result = verify_all_backups(datastore, worker.clone(), worker.upid(), &filter);
+            let job_result = match result {
+                Ok(ref errors) if errors.is_empty() => Ok(()),
+                Ok(_) => Err(format_err!("verification failed - please check the log for details")),
+                Err(_) => Err(format_err!("verification failed - job aborted")),
+            };
+
+            let status = worker.create_state(&job_result);
+
+            match job.finish(status) {
+                Err(err) => eprintln!(
+                    "could not finish job state for {}: {}",
+                    job.jobtype().to_string(),
+                    err
+                ),
+                Ok(_) => (),
+            }
+
+            if let Some(email) = email {
+                if let Err(err) = crate::server::send_verify_status(&email, verification_job, &result) {
+                    eprintln!("send verify notification failed: {}", err);
+                }
+            }
+
+            job_result
+        },
+    )?;
+    Ok(upid_str)
+}

src/server/worker_task.rs

@@ -1,6 +1,5 @@
 use std::collections::{HashMap, VecDeque};
 use std::fs::File;
-use std::path::Path;
 use std::io::{Read, Write, BufRead, BufReader};
 use std::panic::UnwindSafe;
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -21,7 +20,7 @@ use proxmox::tools::fs::{create_path, open_file_locked, replace_file, CreateOpti
 use super::UPID;
 
 use crate::tools::logrotate::{LogRotate, LogRotateFiles};
-use crate::tools::FileLogger;
+use crate::tools::{FileLogger, FileLogOptions};
 use crate::api2::types::Userid;
 
 macro_rules! PROXMOX_BACKUP_VAR_RUN_DIR_M { () => ("/run/proxmox-backup") }
@@ -93,22 +92,27 @@ pub fn create_task_control_socket() -> Result<(), Error> {
         "\0{}/proxmox-task-control-{}.sock", PROXMOX_BACKUP_VAR_RUN_DIR, *MY_PID);
 
     let control_future = super::create_control_socket(socketname, |param| {
-        let param = param.as_object()
+        let param = param
+            .as_object()
             .ok_or_else(|| format_err!("unable to parse parameters (expected json object)"))?;
         if param.keys().count() != 2 { bail!("wrong number of parameters"); }
 
-        let command = param["command"].as_str()
+        let command = param["command"]
+            .as_str()
             .ok_or_else(|| format_err!("unable to parse parameters (missing command)"))?;
 
         // we have only two commands for now
-        if !(command == "abort-task" || command == "status") { bail!("got unknown command '{}'", command); }
+        if !(command == "abort-task" || command == "status") {
+            bail!("got unknown command '{}'", command);
+        }
 
-        let upid_str = param["upid"].as_str()
+        let upid_str = param["upid"]
+            .as_str()
             .ok_or_else(|| format_err!("unable to parse parameters (missing upid)"))?;
 
         let upid = upid_str.parse::<UPID>()?;
 
-        if !((upid.pid == *MY_PID) && (upid.pstart == *MY_PID_PSTART)) {
+        if !(upid.pid == *MY_PID && upid.pstart == *MY_PID_PSTART) {
             bail!("upid does not belong to this process");
         }
 
@@ -344,21 +348,11 @@ fn lock_task_list_files(exclusive: bool) -> Result<std::fs::File, Error> {
 /// rotates it if it is
 pub fn rotate_task_log_archive(size_threshold: u64, compress: bool, max_files: Option<usize>) -> Result<bool, Error> {
     let _lock = lock_task_list_files(true)?;
-    let path = Path::new(PROXMOX_BACKUP_ARCHIVE_TASK_FN);
-    let metadata = path.metadata()?;
-    if metadata.len() > size_threshold {
-        let mut logrotate = LogRotate::new(PROXMOX_BACKUP_ARCHIVE_TASK_FN, compress).ok_or_else(|| format_err!("could not get archive file names"))?;
-        let backup_user = crate::backup::backup_user()?;
-        logrotate.rotate(
-            CreateOptions::new()
-                .owner(backup_user.uid)
-                .group(backup_user.gid),
-            max_files,
-        )?;
-        Ok(true)
-    } else {
-        Ok(false)
-    }
+
+    let mut logrotate = LogRotate::new(PROXMOX_BACKUP_ARCHIVE_TASK_FN, compress)
+        .ok_or(format_err!("could not get archive file names"))?;
+
+    logrotate.rotate(size_threshold, None, max_files)
 }
 
 // atomically read/update the task list, update status of finished tasks
@@ -555,7 +549,8 @@ impl TaskListInfoIterator {
         let archive = if active_only {
             None
         } else {
-            let logrotate = LogRotate::new(PROXMOX_BACKUP_ARCHIVE_TASK_FN, true).ok_or_else(|| format_err!("could not get archive file names"))?;
+            let logrotate = LogRotate::new(PROXMOX_BACKUP_ARCHIVE_TASK_FN, true)
+                .ok_or_else(|| format_err!("could not get archive file names"))?;
             Some(logrotate.files())
         };
 
@@ -667,7 +662,14 @@ impl WorkerTask {
 
         println!("FILE: {:?}", path);
 
-        let logger = FileLogger::new(&path, to_stdout)?;
+        let logger_options = FileLogOptions {
+            to_stdout: to_stdout,
+            exclusive: true,
+            prefix_time: true,
+            read: true,
+            ..Default::default()
+        };
+        let logger = FileLogger::new(&path, logger_options)?;
         nix::unistd::chown(&path, Some(backup_user.uid), Some(backup_user.gid))?;
 
         let worker = Arc::new(Self {
@@ -846,3 +848,19 @@ impl WorkerTask {
         &self.upid
     }
 }
+
+impl crate::task::TaskState for WorkerTask {
+    fn check_abort(&self) -> Result<(), Error> {
+        self.fail_on_abort()
+    }
+
+    fn log(&self, level: log::Level, message: &std::fmt::Arguments) {
+        match level {
+            log::Level::Error => self.warn(&message.to_string()),
+            log::Level::Warn => self.warn(&message.to_string()),
+            log::Level::Info => self.log(&message.to_string()),
+            log::Level::Debug => self.log(&format!("DEBUG: {}", message)),
+            log::Level::Trace => self.log(&format!("TRACE: {}", message)),
+        }
+    }
+}

src/task.rs

@@ -0,0 +1,56 @@
+use anyhow::Error;
+
+/// `WorkerTask` methods commonly used from contexts otherwise not related to the API server.
+pub trait TaskState {
+    /// If the task should be aborted, this should fail with a reasonable error message.
+    fn check_abort(&self) -> Result<(), Error>;
+
+    /// Create a log message for this task.
+    fn log(&self, level: log::Level, message: &std::fmt::Arguments);
+}
+
+/// Convenience implementation:
+impl<T: TaskState + ?Sized> TaskState for std::sync::Arc<T> {
+    fn check_abort(&self) -> Result<(), Error> {
+        <T as TaskState>::check_abort(&*self)
+    }
+
+    fn log(&self, level: log::Level, message: &std::fmt::Arguments) {
+        <T as TaskState>::log(&*self, level, message)
+    }
+}
+
+#[macro_export]
+macro_rules! task_error {
+    ($task:expr, $($fmt:tt)+) => {{
+        $crate::task::TaskState::log(&*$task, log::Level::Error, &format_args!($($fmt)+))
+    }};
+}
+
+#[macro_export]
+macro_rules! task_warn {
+    ($task:expr, $($fmt:tt)+) => {{
+        $crate::task::TaskState::log(&*$task, log::Level::Warn, &format_args!($($fmt)+))
+    }};
+}
+
+#[macro_export]
+macro_rules! task_log {
+    ($task:expr, $($fmt:tt)+) => {{
+        $crate::task::TaskState::log(&*$task, log::Level::Info, &format_args!($($fmt)+))
+    }};
+}
+
+#[macro_export]
+macro_rules! task_debug {
+    ($task:expr, $($fmt:tt)+) => {{
+        $crate::task::TaskState::log(&*$task, log::Level::Debug, &format_args!($($fmt)+))
+    }};
+}
+
+#[macro_export]
+macro_rules! task_trace {
+    ($task:expr, $($fmt:tt)+) => {{
+        $crate::task::TaskState::log(&*$task, log::Level::Trace, &format_args!($($fmt)+))
+    }};
+}

src/tools.rs

@@ -33,6 +33,12 @@ pub mod statistics;
 pub mod systemd;
 pub mod nom;
 pub mod logrotate;
+pub mod loopdev;
+pub mod fuse_loop;
+pub mod socket;
+pub mod subscription;
+pub mod zip;
+pub mod http;
 
 mod parallel_handler;
 pub use parallel_handler::*;
@@ -40,6 +46,10 @@ pub use parallel_handler::*;
 mod wrapped_reader_stream;
 pub use wrapped_reader_stream::*;
 
+mod async_channel_writer;
+pub use async_channel_writer::*;
+
+
 mod std_channel_writer;
 pub use std_channel_writer::*;
 
@@ -315,10 +325,12 @@ pub fn md5sum(data: &[u8]) -> Result<DigestBytes, Error> {
 pub fn get_hardware_address() -> Result<String, Error> {
     static FILENAME: &str = "/etc/ssh/ssh_host_rsa_key.pub";
 
-    let contents = proxmox::tools::fs::file_get_contents(FILENAME)?;
-    let digest = md5sum(&contents)?;
+    let contents = proxmox::tools::fs::file_get_contents(FILENAME)
+        .map_err(|e| format_err!("Error getting host key - {}", e))?;
+    let digest = md5sum(&contents)
+        .map_err(|e| format_err!("Error digesting host key - {}", e))?;
 
-    Ok(proxmox::tools::bin_to_hex(&digest))
+    Ok(proxmox::tools::bin_to_hex(&digest).to_uppercase())
 }
 
 pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {

src/tools/acl.rs

@@ -43,8 +43,8 @@ pub const ACL_NEXT_ENTRY: c_int  = 1;
 
 // acl to extended attribute names constants
 // from: acl/include/acl_ea.h
-pub const ACL_EA_ACCESS: &'static str = "system.posix_acl_access";
-pub const ACL_EA_DEFAULT: &'static str = "system.posix_acl_default";
+pub const ACL_EA_ACCESS: &str = "system.posix_acl_access";
+pub const ACL_EA_DEFAULT: &str = "system.posix_acl_default";
 pub const ACL_EA_VERSION: u32 = 0x0002;
 
 #[link(name = "acl")]