bump version to 0.8.15-1

We need to filter out benchmark group earlier

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "proxmox-backup"
-version = "0.8.11"
+version = "0.8.15"
 authors = ["Dietmar Maurer <dietmar@proxmox.com>"]
 edition = "2018"
 license = "AGPL-3"
@@ -39,11 +39,11 @@ pam-sys = "0.5"
 percent-encoding = "2.1"
 pin-utils = "0.1.0"
 pathpatterns = "0.1.2"
-proxmox = { version = "0.3.3", features = [ "sortable-macro", "api-macro", "websocket" ] }
+proxmox = { version = "0.3.5", features = [ "sortable-macro", "api-macro", "websocket" ] }
 #proxmox = { git = "ssh://gitolite3@proxdev.maurer-it.com/rust/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
 #proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro", "websocket" ] }
 proxmox-fuse = "0.1.0"
-pxar = { version = "0.3.0", features = [ "tokio-io", "futures-io" ] }
+pxar = { version = "0.6.0", features = [ "tokio-io", "futures-io" ] }
 #pxar = { path = "../pxar", features = [ "tokio-io", "futures-io" ] }
 regex = "1.2"
 rustyline = "6"

Makefile

@@ -150,4 +150,4 @@ upload: ${SERVER_DEB} ${CLIENT_DEB} ${DOC_DEB}
 	# check if working directory is clean
 	git diff --exit-code --stat && git diff --exit-code --stat --staged
 	tar cf - ${SERVER_DEB} ${SERVER_DBG_DEB} ${DOC_DEB} | ssh -X repoman@repo.proxmox.com upload --product pbs --dist buster
-	tar cf - ${CLIENT_DEB} ${CLIENT_DBG_DEB} | ssh -X repoman@repo.proxmox.com upload --product "pbs,pve" --dist buster
+	tar cf - ${CLIENT_DEB} ${CLIENT_DBG_DEB} | ssh -X repoman@repo.proxmox.com upload --product "pbs,pve,pmg" --dist buster

debian/changelog

@@ -1,3 +1,66 @@
+rust-proxmox-backup (0.8.15-1) unstable; urgency=medium
+
+  *  verify: skip benchmark directory
+
+  * add benchmark flag to backup creation for proper cleanup when running
+    a benchmark
+
+  * mount: fix mount subcommand
+
+  * ui: only mark backup encrypted if there are any files
+
+  * fix #2983: improve tcp performance
+
+  * improve ui and docs
+
+  * verify: rename corrupted chunks with .bad extension
+
+  * gc: remove .bad files on garbage collect
+
+  * ui: add translation support
+
+  * server/worker_task: fix upid_read_status
+
+  * tools/systemd/time: enable dates for calendarevents
+
+  * server/worker_task: fix 'unknown' status for some big task logs
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 10 Sep 2020 09:25:59 +0200
+
+rust-proxmox-backup (0.8.14-1) unstable; urgency=medium
+
+  * verify speed up: use separate IO thread, use datastore-wide cache (instead
+    of per group)
+
+  * ui: datastore content: improve encrypted column
+
+  * ui: datastore content: show more granular verify state, especially for
+    backup group rows
+
+  * verify: log progress in percent
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 02 Sep 2020 09:36:47 +0200
+
+rust-proxmox-backup (0.8.13-1) unstable; urgency=medium
+
+  * improve and add to documentation
+
+  * save last verify result in snapshot manifest and show it in the GUI
+
+  * gc: use human readable units for summary in task log
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 27 Aug 2020 16:12:07 +0200
+
+rust-proxmox-backup (0.8.12-1) unstable; urgency=medium
+
+  * verify: speedup - only verify chunks once
+
+  * verify: sort backup groups
+
+  * bump pxar dep to 0.4.0
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 25 Aug 2020 08:55:52 +0200
+
 rust-proxmox-backup (0.8.11-1) unstable; urgency=medium
 
   * improve sync jobs, allow to stop them and better logging
@@ -409,4 +472,3 @@ proxmox-backup (0.1-1) unstable; urgency=medium
   * first try
 
  -- Proxmox Support Team <support@proxmox.com>  Fri, 30 Nov 2018 13:03:28 +0100
-

debian/control

@@ -34,14 +34,14 @@ Build-Depends: debhelper (>= 11),
  librust-pathpatterns-0.1+default-dev (>= 0.1.2-~~),
  librust-percent-encoding-2+default-dev (>= 2.1-~~),
  librust-pin-utils-0.1+default-dev,
- librust-proxmox-0.3+api-macro-dev (>= 0.3.3-~~),
+ librust-proxmox-0.3+api-macro-dev (>= 0.3.5-~~),
- librust-proxmox-0.3+default-dev (>= 0.3.3-~~),
+ librust-proxmox-0.3+default-dev (>= 0.3.5-~~),
- librust-proxmox-0.3+sortable-macro-dev (>= 0.3.3-~~),
+ librust-proxmox-0.3+sortable-macro-dev (>= 0.3.5-~~),
- librust-proxmox-0.3+websocket-dev (>= 0.3.3-~~),
+ librust-proxmox-0.3+websocket-dev (>= 0.3.5-~~),
  librust-proxmox-fuse-0.1+default-dev,
- librust-pxar-0.3+default-dev,
+ librust-pxar-0.6+default-dev,
- librust-pxar-0.3+futures-io-dev,
+ librust-pxar-0.6+futures-io-dev,
- librust-pxar-0.3+tokio-io-dev,
+ librust-pxar-0.6+tokio-io-dev,
  librust-regex-1+default-dev (>= 1.2-~~),
  librust-rustyline-6+default-dev,
  librust-serde-1+default-dev,
@@ -103,6 +103,7 @@ Depends: fonts-font-awesome,
          libjs-extjs (>= 6.0.1),
          libzstd1 (>= 1.3.8),
          lvm2,
+         pbs-i18n,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
          proxmox-widget-toolkit (>= 2.2-4),

debian/control.in

@@ -4,6 +4,7 @@ Depends: fonts-font-awesome,
          libjs-extjs (>= 6.0.1),
          libzstd1 (>= 1.3.8),
          lvm2,
+         pbs-i18n,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
          proxmox-widget-toolkit (>= 2.2-4),

debian/postinst

@@ -15,11 +15,10 @@ case "$1" in
 	fi
 	deb-systemd-invoke $_dh_action proxmox-backup.service proxmox-backup-proxy.service >/dev/null || true
 
-	if test -n "$2"; then
+	# FIXME: Remove in future version once we're sure no broken entries remain in anyone's files
-		if dpkg --compare-versions "$2" 'le' '0.8.10-1'; then
+	if grep -q -e ':termproxy::[^@]\+: ' /var/log/proxmox-backup/tasks/active; then
-			echo "Fixing up termproxy user id in task log..."
+	    echo "Fixing up termproxy user id in task log..."
-			flock -w 30 /var/log/proxmox-backup/tasks/active.lock sed -i 's/:termproxy::root: /:termproxy::root@pam: /' /var/log/proxmox-backup/tasks/active
+	    flock -w 30 /var/log/proxmox-backup/tasks/active.lock sed -i 's/:termproxy::\([^@]\+\): /:termproxy::\1@pam: /' /var/log/proxmox-backup/tasks/active
-		fi
 	fi
     ;;
 

docs/Makefile

@@ -28,7 +28,6 @@ COMPILEDIR := ../target/debug
 SPHINXOPTS    += -t devbuild
 endif
 
-
 # Sphinx internal variables.
 ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) .
 
@@ -68,6 +67,12 @@ proxmox-backup-manager.1: proxmox-backup-manager/man1.rst  proxmox-backup-manage
 proxmox-backup-proxy.1: proxmox-backup-proxy/man1.rst  proxmox-backup-proxy/description.rst
 	rst2man $< >$@
 
+.PHONY: onlinehelpinfo
+onlinehelpinfo:
+	@echo "Generating OnlineHelpInfo.js..."
+	$(SPHINXBUILD) -b proxmox-scanrefs $(ALLSPHINXOPTS) $(BUILDDIR)/scanrefs
+	@echo "Build finished. OnlineHelpInfo.js is in $(BUILDDIR)/scanrefs."
+
 .PHONY: html
 html: ${GENERATED_SYNOPSIS}
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html

docs/_ext/proxmox-scanrefs.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+# debugging stuff
+from pprint import pprint
+
+from typing import cast
+
+import json
+import re
+
+import os
+import io
+from docutils import nodes
+
+from sphinx.builders import Builder
+from sphinx.util import logging
+
+logger = logging.getLogger(__name__)
+
+# refs are added in the following manner before the title of a section (note underscore and newline before title):
+# .. _my-label:
+#
+# Section to ref
+# --------------
+#
+#
+# then referred to like (note missing underscore):
+# "see :ref:`my-label`"
+#
+# the benefit of using this is if a label is explicitly set for a section,
+# we can refer to it with this anchor #my-label in the html,
+# even if the section name changes.
+#
+# see https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref
+
+def scan_extjs_files(wwwdir="../www"): # a bit rough i know, but we can optimize later
+    js_files = []
+    used_anchors = []
+    logger.info("scanning extjs files for onlineHelp definitions")
+    for root, dirs, files in os.walk("{}".format(wwwdir)):
+        #print(root, dirs, files)
+        for filename in files:
+            if filename.endswith('.js'):
+                js_files.append(os.path.join(root, filename))
+    for js_file in js_files:
+        fd = open(js_file).read()
+        match = re.search("onlineHelp:\s*[\'\"](.*?)[\'\"]", fd) # match object is tuple
+        if match:
+            anchor = match.groups()[0]
+            anchor = re.sub('_', '-', anchor) # normalize labels
+            logger.info("found onlineHelp: {} in {}".format(anchor, js_file))
+            used_anchors.append(anchor)
+    return used_anchors
+
+
+def setup(app):
+    logger.info('Mapping reference labels...')
+    app.add_builder(ReflabelMapper)
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }
+
+class ReflabelMapper(Builder):
+    name = 'proxmox-scanrefs'
+
+    def init(self):
+        self.docnames = []
+        self.env.online_help = {}
+        self.env.online_help['pbs_documentation_index'] = {
+            'link': '/docs/index.html',
+            'title': 'Proxmox Backup Server Documentation Index',
+        }
+        self.env.used_anchors = scan_extjs_files()
+
+        if not os.path.isdir(self.outdir):
+            os.mkdir(self.outdir)
+
+        self.output_filename = os.path.join(self.outdir, 'OnlineHelpInfo.js')
+        self.output = io.open(self.output_filename, 'w', encoding='UTF-8')
+
+    def write_doc(self, docname, doctree):
+            for node in doctree.traverse(nodes.section):
+                #pprint(vars(node))
+
+                if hasattr(node, 'expect_referenced_by_id') and len(node['ids']) > 1: # explicit labels
+                    filename = self.env.doc2path(docname)
+                    filename_html = re.sub('.rst', '.html', filename)
+                    labelid = node['ids'][1] # [0] is predefined by sphinx, we need [1] for explicit ones
+                    title = cast(nodes.title, node[0])
+                    logger.info('traversing section {}'.format(title.astext()))
+                    ref_name = getattr(title, 'rawsource', title.astext())
+
+                    self.env.online_help[labelid] = {'link': '', 'title': ''}
+                    self.env.online_help[labelid]['link'] = "/docs/" + os.path.basename(filename_html) + "#{}".format(labelid)
+                    self.env.online_help[labelid]['title'] = ref_name
+
+            return
+
+
+    def get_outdated_docs(self):
+        return 'all documents'
+
+    def prepare_writing(self, docnames):
+        return
+
+    def get_target_uri(self, docname, typ=None):
+        return ''
+
+    def validate_anchors(self):
+        #pprint(self.env.online_help)
+        to_remove = []
+        for anchor in self.env.used_anchors:
+            if anchor not in self.env.online_help:
+                logger.info("[-] anchor {} is missing from onlinehelp!".format(anchor))
+        for anchor in self.env.online_help:
+            if anchor not in self.env.used_anchors and anchor != 'pbs_documentation_index':
+                logger.info("[*] anchor {} not used! deleting...".format(anchor))
+                to_remove.append(anchor)
+        for anchor in to_remove:
+            self.env.online_help.pop(anchor, None)
+        return
+
+    def finish(self):
+        # generate OnlineHelpInfo.js output
+        self.validate_anchors()
+
+        self.output.write("const proxmoxOnlineHelpInfo = ")
+        self.output.write(json.dumps(self.env.online_help, indent=2))
+        self.output.write(";\n")
+        self.output.close()
+        return

docs/administration-guide.rst

@@ -24,6 +24,13 @@ good deduplication rates for file archives.
 The Proxmox Backup Server supports both strategies.
 
 
+Image Archives: ``<name>.img``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This is used for virtual machine images and other large binary
+data. Content is split into fixed-sized chunks.
+
+
 File Archives: ``<name>.pxar``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -34,13 +41,6 @@ the :ref:`pxar-format`, split into variable-sized chunks. The format
 is optimized to achieve good deduplication rates.
 
 
-Image Archives: ``<name>.img``
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-This is used for virtual machine images and other large binary
-data. Content is split into fixed-sized chunks.
-
-
 Binary Data (BLOBs)
 ^^^^^^^^^^^^^^^^^^^
 
@@ -146,12 +146,109 @@ when setting up the backup server.
    filesystem configuration from being supported for a datastore. For example,
    ``ext3`` as a whole or ``ext4`` with the ``dir_nlink`` feature manually disabled.
 
+Disk Management
+~~~~~~~~~~~~~~~
+
+.. image:: images/screenshots/pbs-gui-disks.png
+  :width: 230
+  :align: right
+  :alt: List of disks
+
+Proxmox Backup Server comes with a set of disk utilities, which are
+accessed using the ``disk`` subcommand. This subcommand allows you to initialize
+disks, create various filesystems, and get information about the disks.
+
+To view the disks connected to the system, navigate to **Administration ->
+Disks** in the web interface or use the ``list`` subcommand of
+``disk``:
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk list
+  ┌──────┬────────┬─────┬───────────┬─────────────┬───────────────┬─────────┬────────┐
+  │ name │ used   │ gpt │ disk-type │        size │ model         │ wearout │ status │
+  ╞══════╪════════╪═════╪═══════════╪═════════════╪═══════════════╪═════════╪════════╡
+  │ sda  │ lvm    │   1 │ hdd       │ 34359738368 │ QEMU_HARDDISK │       - │ passed │
+  ├──────┼────────┼─────┼───────────┼─────────────┼───────────────┼─────────┼────────┤
+  │ sdb  │ unused │   1 │ hdd       │ 68719476736 │ QEMU_HARDDISK │       - │ passed │
+  ├──────┼────────┼─────┼───────────┼─────────────┼───────────────┼─────────┼────────┤
+  │ sdc  │ unused │   1 │ hdd       │ 68719476736 │ QEMU_HARDDISK │       - │ passed │
+  └──────┴────────┴─────┴───────────┴─────────────┴───────────────┴─────────┴────────┘
+
+To initialize a disk with a new GPT, use the ``initialize`` subcommand:
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk initialize sdX
+
+.. image:: images/screenshots/pbs-gui-disks-dir-create.png
+  :width: 230
+  :align: right
+  :alt: Create a directory
+
+You can create an ``ext4`` or ``xfs`` filesystem on a disk using ``fs
+create``, or by navigating to **Administration -> Disks -> Directory** in the
+web interface and creating one from there. The following command creates an
+``ext4`` filesystem and passes the ``--add-datastore`` parameter, in order to
+automatically create a datastore on the disk (in this case ``sdd``). This will
+create a datastore at the location ``/mnt/datastore/store1``:
+
+|
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk fs create store1 --disk sdd --filesystem ext4 --add-datastore true
+  create datastore 'store1' on disk sdd
+  Percentage done: 1
+  ...
+  Percentage done: 99
+  TASK OK
+
+.. image:: images/screenshots/pbs-gui-disks-zfs-create.png
+  :width: 230
+  :align: right
+  :alt: Create a directory
+
+You can also create a ``zpool`` with various raid levels from **Administration
+-> Disks -> Zpool** in the web interface, or by using ``zpool create``. The command
+below creates a mirrored ``zpool`` using two disks (``sdb`` & ``sdc``) and
+mounts it on the root directory (default):
+
+|
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk zpool create zpool1 --devices sdb,sdc --raidlevel mirror
+  create Mirror zpool 'zpool1' on devices 'sdb,sdc'
+  # "zpool" "create" "-o" "ashift=12" "zpool1" "mirror" "sdb" "sdc"
+
+  TASK OK
+
+.. note::
+  You can also pass the ``--add-datastore`` parameter here, to automatically
+  create a datastore from the disk.
+
+You can use ``disk fs list`` and ``disk zpool list`` to keep track of your
+filesystems and zpools respectively.
+
+If a disk supports S.M.A.R.T. capability, and you have this enabled, you can
+display S.M.A.R.T. attributes from the web interface or by using the command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk smart-attributes sdX
+
 
 Datastore Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~
 
+.. image:: images/screenshots/pbs-gui-datastore.png
+  :width: 230
+  :align: right
+  :alt: Datastore Overview
+
 You can configure multiple datastores. Minimum one datastore needs to be
-configured. The datastore is identified by a simple `name` and points to a
+configured. The datastore is identified by a simple *name* and points to a
 directory on the filesystem. Each datastore also has associated retention
 settings of how many backup snapshots for each interval of ``hourly``,
 ``daily``, ``weekly``, ``monthly``, ``yearly`` as well as a time-independent
@@ -159,13 +256,35 @@ number of backups to keep in that store. :ref:`Pruning <pruning>` and
 :ref:`garbage collection <garbage-collection>` can also be configured to run
 periodically based on a configured :term:`schedule` per datastore.
 
-The following command creates a new datastore called ``store1`` on :file:`/backup/disk1/store1`
+Creating a Datastore
+^^^^^^^^^^^^^^^^^^^^
+.. image:: images/screenshots/pbs-gui-datastore-create-general.png
+  :width: 230
+  :align: right
+  :alt: Create a datastore
+
+You can create a new datastore from the web GUI, by navigating to **Datastore** in
+the menu tree and clicking **Create**. Here:
+
+* *Name* refers to the name of the datastore
+* *Backing Path* is the path to the directory upon which you want to create the
+  datastore
+* *GC Schedule* refers to the time and intervals at which garbage collection
+  runs
+* *Prune Schedule* refers to the frequency at which pruning takes place
+* *Prune Options* set the amount of backups which you would like to keep (see :ref:`Pruning <pruning>`).
+
+Alternatively you can create a new datastore from the command line. The
+following command creates a new datastore called ``store1`` on :file:`/backup/disk1/store1`
 
 .. code-block:: console
 
   # proxmox-backup-manager datastore create store1 /backup/disk1/store1
 
-To list existing datastores run:
+Managing Datastores
+^^^^^^^^^^^^^^^^^^^
+
+To list existing datastores from the command line run:
 
 .. code-block:: console
 
@@ -176,13 +295,15 @@ To list existing datastores run:
   │ store1 │ /backup/disk1/store1 │ This is my default storage. │
   └────────┴──────────────────────┴─────────────────────────────┘
 
-You can change settings of a datastore, for example to set a prune and garbage
+You can change the garbage collection and prune settings of a datastore, by
-collection schedule or retention settings using ``update`` subcommand and view
+editing the datastore from the GUI or by using the ``update`` subcommand. For
-a datastore with the ``show`` subcommand:
+example, the below command changes the garbage collection schedule using the
+``update`` subcommand and prints the properties of the datastore with the
+``show`` subcommand:
 
 .. code-block:: console
 
-  # proxmox-backup-manager datastore update store1 --keep-last 7 --prune-schedule daily --gc-schedule 'Tue 04:27'
+  # proxmox-backup-manager datastore update store1 --gc-schedule 'Tue 04:27'
   # proxmox-backup-manager datastore show store1
   ┌────────────────┬─────────────────────────────┐
   │ Name           │ Value                       │
@@ -260,6 +381,11 @@ directories will store the chunked data after a backup operation has been execut
 User Management
 ~~~~~~~~~~~~~~~
 
+.. image:: images/screenshots/pbs-gui-user-management.png
+  :width: 230
+  :align: right
+  :alt: User management
+
 Proxmox Backup Server supports several authentication realms, and you need to
 choose the realm when you add a new user. Possible realms are:
 
@@ -284,19 +410,22 @@ users:
   │ root@pam    │      1 │        │           │          │                │ Superuser          │
   └─────────────┴────────┴────────┴───────────┴──────────┴────────────────┴────────────────────┘
 
+.. image:: images/screenshots/pbs-gui-user-management-add-user.png
+  :width: 230
+  :align: right
+  :alt: Add a new user
+
 The superuser has full administration rights on everything, so you
-normally want to add other users with less privileges:
+normally want to add other users with less privileges. You can create a new
+user with the ``user create`` subcommand or through the web interface, under
+**Configuration -> User Management**. The ``create`` subcommand lets you specify
+many options like ``--email`` or ``--password``. You can update or change any
+user properties using the ``update`` subcommand later (**Edit** in the GUI):
+
 
 .. code-block:: console
 
   # proxmox-backup-manager user create john@pbs --email john@example.com
-
-The create command lets you specify many options like ``--email`` or
-``--password``. You can update or change any of them using the
-update command later:
-
-.. code-block:: console
-
   # proxmox-backup-manager user update john@pbs --firstname John --lastname Smith
   # proxmox-backup-manager user update john@pbs --comment "An example user."
 
@@ -374,9 +503,15 @@ following roles exist:
 **RemoteSyncOperator**
   Is allowed to read data from a remote.
 
-You can use the ``acl`` subcommand to manage and monitor user permissions. For
+  :width: 230
-example, the command below will add the user ``john@pbs`` as a
+  :align: right
-**DatastoreAdmin** for the data store ``store1``, located at ``/backup/disk1/store1``:
+  :alt: Add permissions for user
+
+You can manage datastore permissions from **Configuration -> Permissions** in
+the web interface. Likewise, you can use the ``acl`` subcommand to manage and
+monitor user permissions from the command line. For example, the command below
+will add the user ``john@pbs`` as a **DatastoreAdmin** for the datastore
+``store1``, located at ``/backup/disk1/store1``:
 
 .. code-block:: console
 
@@ -393,34 +528,126 @@ You can monitor the roles of each user using the following command:
    │ john@pbs │ /datastore/disk1 │         1 │ DatastoreAdmin │
    └──────────┴──────────────────┴───────────┴────────────────┘
 
-A single user can be assigned multiple permission sets for different data stores.
+A single user can be assigned multiple permission sets for different datastores.
 
 .. Note::
-  Naming convention is important here. For data stores on the host,
+  Naming convention is important here. For datastores on the host,
   you must use the convention ``/datastore/{storename}``. For example, to set
-  permissions for a data store mounted at ``/mnt/backup/disk4/store2``, you would use
+  permissions for a datastore mounted at ``/mnt/backup/disk4/store2``, you would use
   ``/datastore/store2`` for the path. For remote stores, use the convention
   ``/remote/{remote}/{storename}``, where ``{remote}`` signifies the name of the
-  remote (see `Remote` below) and ``{storename}`` is the name of the data store on
+  remote (see `Remote` below) and ``{storename}`` is the name of the datastore on
   the remote.
 
+Network Management
+~~~~~~~~~~~~~~~~~~
+
+Proxmox Backup Server provides both a web interface and a command line tool for
+network configuration. You can find the configuration options in the web
+interface under the **Network Interfaces** section of the **Configuration** menu
+tree item. The command line tool is accessed via the ``network`` subcommand.
+These interfaces allow you to carry out some basic network management tasks,
+such as adding, configuring, and removing network interfaces.
+
+.. note:: Any changes made to the network configuration are not
+  applied, until you click on **Apply Configuration** or enter the ``network
+  reload`` command. This allows you to make many changes at once. It also allows
+  you to ensure that your changes are correct before applying them, as making a
+  mistake here can render the server inaccessible over the network.
+
+To get a list of available interfaces, use the following command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network list
+  ┌───────┬────────┬───────────┬────────┬─────────────┬──────────────┬──────────────┐
+  │ name  │ type   │ autostart │ method │ address     │ gateway      │ ports/slaves │
+  ╞═══════╪════════╪═══════════╪════════╪═════════════╪══════════════╪══════════════╡
+  │ bond0 │ bond   │         1 │ static │ x.x.x.x/x   │ x.x.x.x      │ ens18 ens19  │
+  ├───────┼────────┼───────────┼────────┼─────────────┼──────────────┼──────────────┤
+  │ ens18 │ eth    │         1 │ manual │             │              │              │
+  ├───────┼────────┼───────────┼────────┼─────────────┼──────────────┼──────────────┤
+  │ ens19 │ eth    │         1 │ manual │             │              │              │
+  └───────┴────────┴───────────┴────────┴─────────────┴──────────────┴──────────────┘
+
+.. image:: images/screenshots/pbs-gui-network-create-bond.png
+  :width: 230
+  :align: right
+  :alt: Add a network interface
+
+To add a new network interface, use the ``create`` subcommand with the relevant
+parameters. The following command shows a template for creating the bond shown
+in the list above:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network create bond0 --type bond --bond_mode active-backup --slaves ens18,ens19 --autostart true --cidr x.x.x.x/x --gateway x.x.x.x
+
+You can make changes to the configuration of a network interface with the
+``update`` subcommand:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network update bond0 --cidr y.y.y.y/y
+
+You can also remove a network interface:
+
+.. code-block:: console
+
+   # proxmox-backup-manager network remove bond0
+
+The pending changes for the network configuration file will appear at the bottom of the
+web interface. You can also view these changes, by using the command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network changes
+
+If you would like to cancel all changes at this point, you can either click on
+the **Revert** button or use the following command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network revert
+
+If you are happy with the changes and would like to write them into the
+configuration file, select **Apply Configuration**. The corresponding command
+is:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network reload
+
+You can also configure DNS settings, from the **DNS** section
+of **Configuration** or by using the ``dns`` subcommand of
+``proxmox-backup-manager``.
+
 :term:`Remote`
 ~~~~~~~~~~~~~~
 
 A remote refers to a separate Proxmox Backup Server installation and a user on that
 installation, from which you can `sync` datastores to a local datastore with a
-`Sync Job`.
+`Sync Job`. You can configure remotes in the web interface, under **Configuration
+-> Remotes**. Alternatively, you can use the ``remote`` subcommand.
+
+.. image:: images/screenshots/pbs-gui-remote-add.png
+.. image:: images/screenshots/pbs-gui-permissions-add.png
+  :width: 230
+  :align: right
+  :alt: Add a remote
 
 To add a remote, you need its hostname or ip, a userid and password on the
 remote, and its certificate fingerprint. To get the fingerprint, use the
-``proxmox-backup-manager cert info`` command on the remote.
+``proxmox-backup-manager cert info`` command on the remote, or navigate to
+**Dashboard** in the remote's web interface and select **Show Fingerprint**.
 
 .. code-block:: console
 
   # proxmox-backup-manager cert info |grep Fingerprint
   Fingerprint (sha256): 64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe
 
-Using the information specified above, add the remote with:
+Using the information specified above, you can add a remote from the **Remotes**
+configuration panel, or by using the command:
 
 .. code-block:: console
 
@@ -444,10 +671,16 @@ Use the ``list``, ``show``, ``update``, ``remove`` subcommands of
 Sync Jobs
 ~~~~~~~~~
 
-Sync jobs are configured to pull the contents of a datastore on a `Remote` to a
+.. image:: images/screenshots/pbs-gui-syncjob-add.png
-local datastore. You can either start the sync job manually on the GUI or
+  :width: 230
-provide it with a :term:`schedule` to run regularly. The
+  :align: right
-``proxmox-backup-manager sync-job`` command is used to manage sync jobs:
+  :alt: Add a remote
+
+Sync jobs are configured to pull the contents of a datastore on a **Remote** to a
+local datastore. You can either start a sync job manually on the GUI or
+provide it with a :term:`schedule` to run regularly. You can manage sync jobs
+under **Configuration -> Sync Jobs** in the web interface, or using the
+``proxmox-backup-manager sync-job`` command:
 
 .. code-block:: console
 
@@ -462,6 +695,15 @@ provide it with a :term:`schedule` to run regularly. The
   # proxmox-backup-manager sync-job remove pbs2-local
 
 
+Garbage Collection
+~~~~~~~~~~~~~~~~~~
+You can monitor and run :ref:`garbage collection <garbage-collection>` on the
+Proxmox Backup Server using the ``garbage-collection`` subcommand of
+``proxmox-backup-manager``. You can use the ``start`` subcommand to manually start garbage
+collection on an entire datastore and the ``status`` subcommand to see
+attributes relating to the :ref:`garbage collection <garbage-collection>`.
+
+
 Backup Client usage
 -------------------
 
@@ -1068,7 +1310,7 @@ Garbage Collection
 ~~~~~~~~~~~~~~~~~~
 
 The ``prune`` command removes only the backup index files, not the data
-from the data store. This task is left to the garbage collection
+from the datastore. This task is left to the garbage collection
 command. It is recommended to carry out garbage collection on a regular basis.
 
 The garbage collection works in two phases. In the first phase, all
@@ -1143,6 +1385,10 @@ benchmark using the ``benchmark`` subcommand of ``proxmox-backup-client``:
   │ AES256 GCM encryption speed       │ 3974.03 MB/s (104%) │
   └───────────────────────────────────┴─────────────────────┘
 
+.. note:: The percentages given in the output table correspond to a
+  comparison against a Ryzen 7 2700X. The TLS test connects to the
+  local host, so there is no network involved.
+
 You can also pass the ``--output-format`` parameter to output stats in ``json``,
 rather than the default table format.
 
@@ -1161,6 +1407,10 @@ as ``user1@pbs``.
   # pvesm add pbs store2 --server localhost --datastore store2
   # pvesm set store2 --username user1@pbs --password <secret>
 
+.. note:: If you would rather not pass your password as plain text, you can pass
+  the ``--password`` parameter, without any arguments. This will cause the
+  program to prompt you for a password upon entering the command.
+
 If your backup server uses a self signed certificate, you need to add
 the certificate fingerprint to the configuration. You can get the
 fingerprint by running the following command on the backup server:

docs/conf.py

@@ -18,9 +18,12 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-# import sys
+import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
+# custom extensions
+sys.path.append(os.path.abspath("./_ext"))
+
 # -- Implement custom formatter for code-blocks ---------------------------
 #
 # * use smaller font
@@ -46,7 +49,7 @@ PygmentsBridge.latex_formatter = CustomLatexFormatter
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 
-extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo"]
+extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo", "proxmox-scanrefs"]
 
 todo_link_only = True
 

docs/epilog.rst

@@ -13,7 +13,8 @@
 .. _Proxmox: https://www.proxmox.com
 .. _Proxmox Community Forum: https://forum.proxmox.com
 .. _Proxmox Virtual Environment: https://www.proxmox.com/proxmox-ve
-.. _Proxmox Backup: https://pbs.proxmox.com/wiki/index.php/Main_Page  // FIXME
+  // FIXME
+.. _Proxmox Backup: https://pbs.proxmox.com/wiki/index.php/Main_Page
 .. _PBS Development List: https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
 .. _reStructuredText: https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html
 .. _Rust: https://www.rust-lang.org/

docs/introduction.rst

@@ -22,7 +22,7 @@ Architecture
 ------------
 
 Proxmox Backup Server uses a `client-server model`_. The server stores the
-backup data and provides an API to create and manage data stores. With the
+backup data and provides an API to create and manage datastores. With the
 API, it's also possible to manage disks and other server-side resources.
 
 The backup client uses this API to access the backed up data. With the command

docs/local-zfs.rst

@@ -1,3 +1,6 @@
+
+.. _chapter-zfs:
+
 ZFS on Linux
 ------------
 

src/api2/admin/datastore.rs

@@ -1,6 +1,7 @@
 use std::collections::{HashSet, HashMap};
 use std::ffi::OsStr;
 use std::os::unix::ffi::OsStrExt;
+use std::sync::{Arc, Mutex};
 
 use anyhow::{bail, format_err, Error};
 use futures::*;
@@ -361,7 +362,7 @@ pub fn list_snapshots (
 
         let mut size = None;
 
-        let (comment, files) = match get_all_snapshot_files(&datastore, &info) {
+        let (comment, verification, files) = match get_all_snapshot_files(&datastore, &info) {
             Ok((manifest, files)) => {
                 size = Some(files.iter().map(|x| x.size.unwrap_or(0)).sum());
                 // extract the first line from notes
@@ -370,11 +371,21 @@ pub fn list_snapshots (
                     .and_then(|notes| notes.lines().next())
                     .map(String::from);
 
-                (comment, files)
+                let verify = manifest.unprotected["verify_state"].clone();
+                let verify: Option<SnapshotVerifyState> = match serde_json::from_value(verify) {
+                    Ok(verify) => verify,
+                    Err(err) => {
+                        eprintln!("error parsing verification state : '{}'", err);
+                        None
+                    }
+                };
+
+                (comment, verify, files)
             },
             Err(err) => {
                 eprintln!("error during snapshot file listing: '{}'", err);
                 (
+                    None,
                     None,
                     info
                         .files
@@ -394,6 +405,7 @@ pub fn list_snapshots (
             backup_id: group.backup_id().to_string(),
             backup_time: info.backup_dir.backup_time().timestamp(),
             comment,
+            verification,
             files,
             size,
             owner: Some(owner),
@@ -489,7 +501,7 @@ pub fn verify(
         (None, None, None) => {
             worker_id = store.clone();
         }
-        _ => bail!("parameters do not spefify a backup group or snapshot"),
+        _ => bail!("parameters do not specify a backup group or snapshot"),
     }
 
     let userid: Userid = rpcenv.get_user().unwrap().parse()?;
@@ -501,25 +513,34 @@ pub fn verify(
         userid,
         to_stdout,
         move |worker| {
+            let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*16)));
+            let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
+
             let failed_dirs = if let Some(backup_dir) = backup_dir {
-                let mut verified_chunks = HashSet::with_capacity(1024*16);
-                let mut corrupt_chunks = HashSet::with_capacity(64);
                 let mut res = Vec::new();
-                if !verify_backup_dir(&datastore, &backup_dir, &mut verified_chunks, &mut corrupt_chunks, &worker)? {
+                if !verify_backup_dir(datastore, &backup_dir, verified_chunks, corrupt_chunks, worker.clone())? {
                     res.push(backup_dir.to_string());
                 }
                 res
             } else if let Some(backup_group) = backup_group {
-                verify_backup_group(&datastore, &backup_group, &worker)?
+                let (_count, failed_dirs) = verify_backup_group(
+                    datastore,
+                    &backup_group,
+                    verified_chunks,
+                    corrupt_chunks,
+                    None,
+                    worker.clone(),
+                )?;
+                failed_dirs
             } else {
-                verify_all_backups(&datastore, &worker)?
+                verify_all_backups(datastore, worker.clone())?
             };
             if failed_dirs.len() > 0 {
                 worker.log("Failed to verify following snapshots:");
                 for dir in failed_dirs {
                     worker.log(format!("\t{}", dir));
                 }
-                bail!("verfication failed - please check the log for details");
+                bail!("verification failed - please check the log for details");
             }
             Ok(())
         },
@@ -1218,7 +1239,7 @@ fn catalog(
 pub const API_METHOD_PXAR_FILE_DOWNLOAD: ApiMethod = ApiMethod::new(
     &ApiHandler::AsyncHttp(&pxar_file_download),
     &ObjectSchema::new(
-        "Download single file from pxar file of a bacup snapshot. Only works if it's not encrypted.",
+        "Download single file from pxar file of a backup snapshot. Only works if it's not encrypted.",
         &sorted!([
             ("store", false, &DATASTORE_SCHEMA),
             ("backup-type", false, &BACKUP_TYPE_SCHEMA),

src/api2/admin/sync.rs

@@ -57,7 +57,8 @@ pub fn list_sync_jobs(
         job.next_run = (|| -> Option<i64> {
             let schedule = job.schedule.as_ref()?;
             let event = parse_calendar_event(&schedule).ok()?;
-            compute_next_event(&event, last, false).ok()
+            // ignore errors
+            compute_next_event(&event, last, false).unwrap_or_else(|_| None)
         })();
     }
 

src/api2/backup.rs

@@ -38,6 +38,7 @@ pub const API_METHOD_UPGRADE_BACKUP: ApiMethod = ApiMethod::new(
             ("backup-id", false, &BACKUP_ID_SCHEMA),
             ("backup-time", false, &BACKUP_TIME_SCHEMA),
             ("debug", true, &BooleanSchema::new("Enable verbose debug logging.").schema()),
+            ("benchmark", true, &BooleanSchema::new("Job is a benchmark (do not keep data).").schema()),
         ]),
     )
 ).access(
@@ -56,6 +57,7 @@ fn upgrade_to_backup_protocol(
 
 async move {
     let debug = param["debug"].as_bool().unwrap_or(false);
+    let benchmark = param["benchmark"].as_bool().unwrap_or(false);
 
     let userid: Userid = rpcenv.get_user().unwrap().parse()?;
 
@@ -90,11 +92,24 @@ async move {
 
     let backup_group = BackupGroup::new(backup_type, backup_id);
 
+    let worker_type = if backup_type == "host" && backup_id == "benchmark" {
+        if !benchmark {
+            bail!("unable to run benchmark without --benchmark flags");
+        }
+        "benchmark"
+    } else {
+        if benchmark {
+            bail!("benchmark flags is only allowed on 'host/benchmark'");
+        }
+        "backup"
+    };
+
     // lock backup group to only allow one backup per group at a time
     let (owner, _group_guard) = datastore.create_locked_backup_group(&backup_group, &userid)?;
 
     // permission check
-    if owner != userid { // only the owner is allowed to create additional snapshots
+    if owner != userid && worker_type != "benchmark" {
+        // only the owner is allowed to create additional snapshots
         bail!("backup owner check failed ({} != {})", userid, owner);
     }
 
@@ -116,14 +131,15 @@ async move {
     let (path, is_new, _snap_guard) = datastore.create_locked_backup_dir(&backup_dir)?;
     if !is_new { bail!("backup directory already exists."); }
 
-    WorkerTask::spawn("backup", Some(worker_id), userid.clone(), true, move |worker| {
+
+    WorkerTask::spawn(worker_type, Some(worker_id), userid.clone(), true, move |worker| {
         let mut env = BackupEnvironment::new(
             env_type, userid, worker.clone(), datastore, backup_dir);
 
         env.debug = debug;
         env.last_backup = last_backup;
 
-        env.log(format!("starting new backup on datastore '{}': {:?}", store, path));
+        env.log(format!("starting new {} on datastore '{}': {:?}", worker_type, store, path));
 
         let service = H2Service::new(env.clone(), worker.clone(), &BACKUP_API_ROUTER, debug);
 
@@ -160,7 +176,11 @@ async move {
                 req = req_fut => req,
                 abrt = abort_future => abrt,
             };
-
+            if benchmark {
+                env.log("benchmark finished successfully");
+                env.remove_backup()?;
+                return Ok(());
+            }
             match (res, env.ensure_finished()) {
                 (Ok(_), Ok(())) => {
                     env.log("backup finished successfully");

src/api2/backup/environment.rs

@@ -457,11 +457,11 @@ impl BackupEnvironment {
     /// Mark backup as finished
     pub fn finish_backup(&self) -> Result<(), Error> {
         let mut state = self.state.lock().unwrap();
-        // test if all writer are correctly closed
 
         state.ensure_unfinished()?;
 
-        if state.dynamic_writers.len() != 0 {
+        // test if all writer are correctly closed
+        if state.dynamic_writers.len() != 0 || state.fixed_writers.len() != 0 {
             bail!("found open index writer - unable to finish backup");
         }
 

src/api2/status.rs

@@ -74,6 +74,9 @@ use crate::config::acl::{
             },
         },
     },
+    access: {
+        permission: &Permission::Anybody,
+    },
 )]
 /// List Datastore usages and estimates
 fn datastore_status(

src/api2/types/mod.rs

@@ -6,6 +6,7 @@ use proxmox::const_regex;
 use proxmox::{IPRE, IPV4RE, IPV6RE, IPV4OCTET, IPV6H16, IPV6LS32};
 
 use crate::backup::CryptMode;
+use crate::server::UPID;
 
 #[macro_use]
 mod macros;
@@ -379,6 +380,25 @@ pub struct GroupListItem {
     pub owner: Option<Userid>,
 }
 
+#[api(
+    properties: {
+        upid: {
+            schema: UPID_SCHEMA
+        },
+        state: {
+            type: String
+        },
+    },
+)]
+#[derive(Serialize, Deserialize)]
+/// Task properties.
+pub struct SnapshotVerifyState {
+    /// UPID of the verify task
+    pub upid: UPID,
+    /// State of the verification. "failed" or "ok"
+    pub state: String,
+}
+
 #[api(
     properties: {
         "backup-type": {
@@ -390,6 +410,14 @@ pub struct GroupListItem {
         "backup-time": {
             schema: BACKUP_TIME_SCHEMA,
         },
+        comment: {
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+            optional: true,
+        },
+        verification: {
+            type: SnapshotVerifyState,
+            optional: true,
+        },
         files: {
             items: {
                 schema: BACKUP_ARCHIVE_NAME_SCHEMA
@@ -411,6 +439,9 @@ pub struct SnapshotListItem {
     /// The first line from manifest "notes"
     #[serde(skip_serializing_if="Option::is_none")]
     pub comment: Option<String>,
+    /// The result of the last run verify task
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub verification: Option<SnapshotVerifyState>,
     /// List of contained archive files.
     pub files: Vec<BackupContent>,
     /// Overall snapshot size (sum of all archive sizes).
@@ -528,6 +559,8 @@ pub struct GarbageCollectionStatus {
     pub pending_bytes: u64,
     /// Number of pending chunks (pending removal - kept for safety).
     pub pending_chunks: usize,
+    /// Number of chunks marked as .bad by verify that have been removed by GC.
+    pub removed_bad: usize,
 }
 
 impl Default for GarbageCollectionStatus {
@@ -542,6 +575,7 @@ impl Default for GarbageCollectionStatus {
             removed_chunks: 0,
             pending_bytes: 0,
             pending_chunks: 0,
+            removed_bad: 0,
         }
     }
 }

src/api2/types/userid.rs

@@ -9,7 +9,7 @@
 //!   with `String`, meaning you can only make references to it.
 //! * [`Realm`]: an owned realm (`String` equivalent).
 //! * [`RealmRef`]: a borrowed realm (`str` equivalent).
-//! * [`Userid`]: an owned user id (`"user@realm"`). Note that this does not have a separte
+//! * [`Userid`]: an owned user id (`"user@realm"`). Note that this does not have a separate
 //!   borrowed type.
 //!
 //! Note that `Username`s are not unique, therefore they do not implement `Eq` and cannot be

src/backup.rs

@@ -120,6 +120,8 @@ macro_rules! PROXMOX_BACKUP_READER_PROTOCOL_ID_V1 {
 
 /// Unix system user used by proxmox-backup-proxy
 pub const BACKUP_USER_NAME: &str = "backup";
+/// Unix system group used by proxmox-backup-proxy
+pub const BACKUP_GROUP_NAME: &str = "backup";
 
 /// Return User info for the 'backup' user (``getpwnam_r(3)``)
 pub fn backup_user() -> Result<nix::unistd::User, Error> {
@@ -129,6 +131,14 @@ pub fn backup_user() -> Result<nix::unistd::User, Error> {
     }
 }
 
+/// Return Group info for the 'backup' group (``getgrnam(3)``)
+pub fn backup_group() -> Result<nix::unistd::Group, Error> {
+    match nix::unistd::Group::from_name(BACKUP_GROUP_NAME)? {
+        Some(group) => Ok(group),
+        None => bail!("Unable to lookup backup user."),
+    }
+}
+
 mod file_formats;
 pub use file_formats::*;
 

src/backup/backup_info.rs

@@ -45,6 +45,31 @@ pub struct BackupGroup {
     backup_id: String,
 }
 
+impl std::cmp::Ord for BackupGroup {
+
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        let type_order = self.backup_type.cmp(&other.backup_type);
+        if type_order != std::cmp::Ordering::Equal {
+            return type_order;
+        }
+        // try to compare IDs numerically
+        let id_self = self.backup_id.parse::<u64>();
+        let id_other = other.backup_id.parse::<u64>();
+        match (id_self, id_other) {
+            (Ok(id_self), Ok(id_other)) => id_self.cmp(&id_other),
+            (Ok(_), Err(_)) => std::cmp::Ordering::Less,
+            (Err(_), Ok(_)) => std::cmp::Ordering::Greater,
+             _ => self.backup_id.cmp(&other.backup_id),
+        }
+    }
+}
+
+impl std::cmp::PartialOrd for BackupGroup {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 impl BackupGroup {
 
     pub fn new<T: Into<String>, U: Into<String>>(backup_type: T, backup_id: U) -> Self {

src/backup/chunk_store.rs

@@ -104,7 +104,7 @@ impl ChunkStore {
             }
             let percentage = (i*100)/(64*1024);
             if percentage != last_percentage {
-                eprintln!("Percentage done: {}", percentage);
+                eprintln!("{}%", percentage);
                 last_percentage = percentage;
             }
         }
@@ -187,7 +187,7 @@ impl ChunkStore {
     pub fn get_chunk_iterator(
         &self,
     ) -> Result<
-        impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize)> + std::iter::FusedIterator,
+        impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize, bool)> + std::iter::FusedIterator,
         Error
     > {
         use nix::dir::Dir;
@@ -219,19 +219,21 @@ impl ChunkStore {
                         Some(Ok(entry)) => {
                             // skip files if they're not a hash
                             let bytes = entry.file_name().to_bytes();
-                            if bytes.len() != 64 {
+                            if bytes.len() != 64 && bytes.len() != 64 + ".0.bad".len() {
                                 continue;
                             }
-                            if !bytes.iter().all(u8::is_ascii_hexdigit) {
+                            if !bytes.iter().take(64).all(u8::is_ascii_hexdigit) {
                                 continue;
                             }
-                            return Some((Ok(entry), percentage));
+
+                            let bad = bytes.ends_with(".bad".as_bytes());
+                            return Some((Ok(entry), percentage, bad));
                         }
                         Some(Err(err)) => {
                             // stop after first error
                             done = true;
                             // and pass the error through:
-                            return Some((Err(err), percentage));
+                            return Some((Err(err), percentage, false));
                         }
                         None => (), // open next directory
                     }
@@ -261,7 +263,7 @@ impl ChunkStore {
                         // other errors are fatal, so end our iteration
                         done = true;
                         // and pass the error through:
-                        return Some((Err(format_err!("unable to read subdir '{}' - {}", subdir, err)), percentage));
+                        return Some((Err(format_err!("unable to read subdir '{}' - {}", subdir, err)), percentage, false));
                     }
                 }
             }
@@ -280,6 +282,7 @@ impl ChunkStore {
         worker: &WorkerTask,
     ) -> Result<(), Error> {
         use nix::sys::stat::fstatat;
+        use nix::unistd::{unlinkat, UnlinkatFlags};
 
         let mut min_atime = phase1_start_time - 3600*24; // at least 24h (see mount option relatime)
 
@@ -292,10 +295,10 @@ impl ChunkStore {
         let mut last_percentage = 0;
         let mut chunk_count = 0;
 
-        for (entry, percentage) in self.get_chunk_iterator()? {
+        for (entry, percentage, bad) in self.get_chunk_iterator()? {
             if last_percentage != percentage {
                 last_percentage = percentage;
-                worker.log(format!("percentage done: {}, chunk count: {}", percentage, chunk_count));
+                worker.log(format!("percentage done: phase2 {}% (processed {} chunks)", percentage, chunk_count));
             }
 
             worker.fail_on_abort()?;
@@ -321,14 +324,47 @@ impl ChunkStore {
             let lock = self.mutex.lock();
 
             if let Ok(stat) = fstatat(dirfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
-                if stat.st_atime < min_atime {
+                if bad {
+                    // filename validity checked in iterator
+                    let orig_filename = std::ffi::CString::new(&filename.to_bytes()[..64])?;
+                    match fstatat(
+                        dirfd,
+                        orig_filename.as_c_str(),
+                        nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW)
+                    {
+                        Ok(_) => {
+                            match unlinkat(Some(dirfd), filename, UnlinkatFlags::NoRemoveDir) {
+                                Err(err) =>
+                                    worker.warn(format!(
+                                        "unlinking corrupt chunk {:?} failed on store '{}' - {}",
+                                        filename,
+                                        self.name,
+                                        err,
+                                    )),
+                                Ok(_) => {
+                                    status.removed_bad += 1;
+                                    status.removed_bytes += stat.st_size as u64;
+                                }
+                            }
+                        },
+                        Err(nix::Error::Sys(nix::errno::Errno::ENOENT)) => {
+                            // chunk hasn't been rewritten yet, keep .bad file
+                        },
+                        Err(err) => {
+                            // some other error, warn user and keep .bad file around too
+                            worker.warn(format!(
+                                "error during stat on '{:?}' - {}",
+                                orig_filename,
+                                err,
+                            ));
+                        }
+                    }
+                } else if stat.st_atime < min_atime {
                     //let age = now - stat.st_atime;
                     //println!("UNLINK {}  {:?}", age/(3600*24), filename);
-                    let res = unsafe { libc::unlinkat(dirfd, filename.as_ptr(), 0) };
+                    if let Err(err) = unlinkat(Some(dirfd), filename, UnlinkatFlags::NoRemoveDir) {
-                    if res != 0 {
-                        let err = nix::Error::last();
                         bail!(
-                            "unlink chunk {:?} failed on store '{}' - {}",
+                            "unlinking chunk {:?} failed on store '{}' - {}",
                             filename,
                             self.name,
                             err,
@@ -366,6 +402,7 @@ impl ChunkStore {
 
         if let Ok(metadata) = std::fs::metadata(&chunk_path) {
             if metadata.is_file() {
+                self.touch_chunk(digest)?;
                 return Ok((true, metadata.len()));
             } else {
                 bail!("Got unexpected file type on store '{}' for chunk {}", self.name, digest_str);

src/backup/data_blob.rs

@@ -304,7 +304,7 @@ impl DataBlob {
 
         let digest = match config {
             Some(config) => config.compute_digest(data),
-            None => openssl::sha::sha256(&data),
+            None => openssl::sha::sha256(data),
         };
         if &digest != expected_digest {
             bail!("detected chunk with wrong digest.");

src/backup/datastore.rs

@@ -21,6 +21,7 @@ use super::{DataBlob, ArchiveType, archive_type};
 use crate::config::datastore;
 use crate::server::WorkerTask;
 use crate::tools;
+use crate::tools::format::HumanByte;
 use crate::tools::fs::{lock_dir_noblock, DirLockGuard};
 use crate::api2::types::{GarbageCollectionStatus, Userid};
 
@@ -84,7 +85,7 @@ impl DataStore {
     pub fn get_chunk_iterator(
         &self,
     ) -> Result<
-        impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize)>,
+        impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize, bool)>,
         Error
     > {
         self.chunk_store.get_chunk_iterator()
@@ -299,7 +300,7 @@ impl DataStore {
     /// And set the owner to 'userid'. If the group already exists, it returns the
     /// current owner (instead of setting the owner).
     ///
-    /// This also aquires an exclusive lock on the directory and returns the lock guard.
+    /// This also acquires an exclusive lock on the directory and returns the lock guard.
     pub fn create_locked_backup_group(
         &self,
         backup_group: &BackupGroup,
@@ -429,6 +430,12 @@ impl DataStore {
 
         let image_list = self.list_images()?;
 
+        let image_count = image_list.len();
+
+        let mut done = 0;
+
+        let mut last_percentage: usize = 0;
+
         for path in image_list {
 
             worker.fail_on_abort()?;
@@ -443,6 +450,14 @@ impl DataStore {
                     self.index_mark_used_chunks(index, &path, status, worker)?;
                 }
             }
+            done += 1;
+
+            let percentage = done*100/image_count;
+            if percentage > last_percentage {
+                worker.log(format!("percentage done: phase1 {}% ({} of {} index files)",
+                                   percentage, done, image_count));
+                last_percentage = percentage;
+            }
         }
 
         Ok(())
@@ -462,9 +477,8 @@ impl DataStore {
 
             let _exclusive_lock =  self.chunk_store.try_exclusive_lock()?;
 
-            let now = unsafe { libc::time(std::ptr::null_mut()) };
+            let phase1_start_time = unsafe { libc::time(std::ptr::null_mut()) };
-
+            let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(phase1_start_time);
-            let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(now);
 
             let mut gc_status = GarbageCollectionStatus::default();
             gc_status.upid = Some(worker.to_string());
@@ -474,26 +488,29 @@ impl DataStore {
             self.mark_used_chunks(&mut gc_status, &worker)?;
 
             worker.log("Start GC phase2 (sweep unused chunks)");
-            self.chunk_store.sweep_unused_chunks(oldest_writer, now, &mut gc_status, &worker)?;
+            self.chunk_store.sweep_unused_chunks(oldest_writer, phase1_start_time, &mut gc_status, &worker)?;
 
-            worker.log(&format!("Removed bytes: {}", gc_status.removed_bytes));
+            worker.log(&format!("Removed garbage: {}", HumanByte::from(gc_status.removed_bytes)));
             worker.log(&format!("Removed chunks: {}", gc_status.removed_chunks));
             if gc_status.pending_bytes > 0 {
-                worker.log(&format!("Pending removals: {} bytes ({} chunks)", gc_status.pending_bytes, gc_status.pending_chunks));
+                worker.log(&format!("Pending removals: {} (in {} chunks)", HumanByte::from(gc_status.pending_bytes), gc_status.pending_chunks));
+            }
+            if gc_status.removed_bad > 0 {
+                worker.log(&format!("Removed bad files: {}", gc_status.removed_bad));
             }
 
-            worker.log(&format!("Original data bytes: {}", gc_status.index_data_bytes));
+            worker.log(&format!("Original data usage: {}", HumanByte::from(gc_status.index_data_bytes)));
 
             if gc_status.index_data_bytes > 0 {
-                let comp_per = (gc_status.disk_bytes*100)/gc_status.index_data_bytes;
+                let comp_per = (gc_status.disk_bytes as f64 * 100.)/gc_status.index_data_bytes as f64;
-                worker.log(&format!("Disk bytes: {} ({} %)", gc_status.disk_bytes, comp_per));
+                worker.log(&format!("On-Disk usage: {} ({:.2}%)", HumanByte::from(gc_status.disk_bytes), comp_per));
             }
 
-            worker.log(&format!("Disk chunks: {}", gc_status.disk_chunks));
+            worker.log(&format!("On-Disk chunks: {}", gc_status.disk_chunks));
 
             if gc_status.disk_chunks > 0 {
                 let avg_chunk = gc_status.disk_bytes/(gc_status.disk_chunks as u64);
-                worker.log(&format!("Average chunk size: {}", avg_chunk));
+                worker.log(&format!("Average chunk size: {}", HumanByte::from(avg_chunk)));
             }
 
             *self.last_gc_status.lock().unwrap() = gc_status;

src/backup/manifest.rs

@@ -145,7 +145,7 @@ impl BackupManifest {
         Ok(())
     }
 
-    // Generate cannonical json
+    // Generate canonical json
     fn to_canonical_json(value: &Value) -> Result<Vec<u8>, Error> {
         let mut data = Vec::new();
         Self::write_canonical_json(value, &mut data)?;

src/backup/verify.rs

@@ -1,16 +1,20 @@
 use std::collections::HashSet;
+use std::sync::{Arc, Mutex};
+use std::sync::atomic::{Ordering, AtomicUsize};
+use std::time::Instant;
 
-use anyhow::{bail, Error};
+use anyhow::{bail, format_err, Error};
 
 use crate::server::WorkerTask;
+use crate::api2::types::*;
 
 use super::{
-    DataStore, BackupGroup, BackupDir, BackupInfo, IndexFile,
+    DataStore, DataBlob, BackupGroup, BackupDir, BackupInfo, IndexFile,
     CryptMode,
     FileInfo, ArchiveType, archive_type,
 };
 
-fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
+fn verify_blob(datastore: Arc<DataStore>, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
 
     let blob = datastore.load_blob(backup_dir, &info.filename)?;
 
@@ -35,38 +39,125 @@ fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -
     }
 }
 
+fn rename_corrupted_chunk(
+    datastore: Arc<DataStore>,
+    digest: &[u8;32],
+    worker: Arc<WorkerTask>,
+) {
+    let (path, digest_str) = datastore.chunk_path(digest);
+
+    let mut counter = 0;
+    let mut new_path = path.clone();
+    loop {
+        new_path.set_file_name(format!("{}.{}.bad", digest_str, counter));
+        if new_path.exists() && counter < 9 { counter += 1; } else { break; }
+    }
+
+    match std::fs::rename(&path, &new_path) {
+        Ok(_) => {
+            worker.log(format!("corrupted chunk renamed to {:?}", &new_path));
+        },
+        Err(err) => {
+            match err.kind() {
+                std::io::ErrorKind::NotFound => { /* ignored */ },
+                _ => worker.log(format!("could not rename corrupted chunk {:?} - {}", &path, err))
+            }
+        }
+    };
+}
+
+// We use a separate thread to read/load chunks, so that we can do
+// load and verify in parallel to increase performance.
+fn chunk_reader_thread(
+    datastore: Arc<DataStore>,
+    index: Box<dyn IndexFile + Send>,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    errors: Arc<AtomicUsize>,
+    worker: Arc<WorkerTask>,
+) -> std::sync::mpsc::Receiver<(DataBlob, [u8;32], u64)> {
+
+    let (sender, receiver) = std::sync::mpsc::sync_channel(3); // buffer up to 3 chunks
+
+    std::thread::spawn(move|| {
+        for pos in 0..index.index_count() {
+            let info = index.chunk_info(pos).unwrap();
+            let size = info.range.end - info.range.start;
+
+            if verified_chunks.lock().unwrap().contains(&info.digest) {
+                continue; // already verified
+            }
+
+            if corrupt_chunks.lock().unwrap().contains(&info.digest) {
+                let digest_str = proxmox::tools::digest_to_hex(&info.digest);
+                worker.log(format!("chunk {} was marked as corrupt", digest_str));
+                errors.fetch_add(1, Ordering::SeqCst);
+                continue;
+            }
+
+            match datastore.load_chunk(&info.digest) {
+                Err(err) => {
+                    corrupt_chunks.lock().unwrap().insert(info.digest);
+                    worker.log(format!("can't verify chunk, load failed - {}", err));
+                    errors.fetch_add(1, Ordering::SeqCst);
+                    rename_corrupted_chunk(datastore.clone(), &info.digest, worker.clone());
+                    continue;
+                }
+                Ok(chunk) => {
+                    if sender.send((chunk, info.digest, size)).is_err() {
+                        break; // receiver gone - simply stop
+                    }
+                }
+            }
+        }
+    });
+
+    receiver
+}
+
 fn verify_index_chunks(
-    datastore: &DataStore,
+    datastore: Arc<DataStore>,
-    index: Box<dyn IndexFile>,
+    index: Box<dyn IndexFile + Send>,
-    verified_chunks: &mut HashSet<[u8;32]>,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
-    corrupt_chunks: &mut HashSet<[u8; 32]>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8; 32]>>>,
     crypt_mode: CryptMode,
-    worker: &WorkerTask,
+    worker: Arc<WorkerTask>,
 ) -> Result<(), Error> {
 
-    let mut errors = 0;
+    let errors = Arc::new(AtomicUsize::new(0));
-    for pos in 0..index.index_count() {
+
+    let start_time = Instant::now();
+
+    let chunk_channel = chunk_reader_thread(
+        datastore.clone(),
+        index,
+        verified_chunks.clone(),
+        corrupt_chunks.clone(),
+        errors.clone(),
+        worker.clone(),
+    );
+
+    let mut read_bytes = 0;
+    let mut decoded_bytes = 0;
+
+    loop {
 
         worker.fail_on_abort()?;
+        crate::tools::fail_on_shutdown()?;
 
-        let info = index.chunk_info(pos).unwrap();
+        let (chunk, digest, size) = match chunk_channel.recv() {
-        let size = info.range.end - info.range.start;
+            Ok(tuple) => tuple,
-
+            Err(std::sync::mpsc::RecvError) => break,
-        let chunk = match datastore.load_chunk(&info.digest) {
-            Err(err) => {
-                corrupt_chunks.insert(info.digest);
-                worker.log(format!("can't verify chunk, load failed - {}", err));
-                errors += 1;
-                continue;
-            },
-            Ok(chunk) => chunk,
         };
 
+        read_bytes += chunk.raw_size();
+        decoded_bytes += size;
+
         let chunk_crypt_mode = match chunk.crypt_mode() {
             Err(err) => {
-                corrupt_chunks.insert(info.digest);
+                corrupt_chunks.lock().unwrap().insert(digest);
                 worker.log(format!("can't verify chunk, unknown CryptMode - {}", err));
-                errors += 1;
+                errors.fetch_add(1, Ordering::SeqCst);
                 continue;
             },
             Ok(mode) => mode,
@@ -78,27 +169,33 @@ fn verify_index_chunks(
                 chunk_crypt_mode,
                 crypt_mode
             ));
-            errors += 1;
+            errors.fetch_add(1, Ordering::SeqCst);
         }
 
-        if !verified_chunks.contains(&info.digest) {
+        if let Err(err) = chunk.verify_unencrypted(size as usize, &digest) {
-            if !corrupt_chunks.contains(&info.digest) {
+            corrupt_chunks.lock().unwrap().insert(digest);
-                if let Err(err) = chunk.verify_unencrypted(size as usize, &info.digest) {
+            worker.log(format!("{}", err));
-                    corrupt_chunks.insert(info.digest);
+            errors.fetch_add(1, Ordering::SeqCst);
-                    worker.log(format!("{}", err));
+            rename_corrupted_chunk(datastore.clone(), &digest, worker.clone());
-                    errors += 1;
+        } else {
-                } else {
+            verified_chunks.lock().unwrap().insert(digest);
-                    verified_chunks.insert(info.digest);
-                }
-            } else {
-                let digest_str = proxmox::tools::digest_to_hex(&info.digest);
-                worker.log(format!("chunk {} was marked as corrupt", digest_str));
-                errors += 1;
-            }
         }
     }
 
-    if errors > 0 {
+    let elapsed = start_time.elapsed().as_secs_f64();
+
+    let read_bytes_mib = (read_bytes as f64)/(1024.0*1024.0);
+    let decoded_bytes_mib = (decoded_bytes as f64)/(1024.0*1024.0);
+
+    let read_speed = read_bytes_mib/elapsed;
+    let decode_speed = decoded_bytes_mib/elapsed;
+
+    let error_count = errors.load(Ordering::SeqCst);
+
+    worker.log(format!("  verified {:.2}/{:.2} MiB in {:.2} seconds, speed {:.2}/{:.2} MiB/s ({} errors)",
+                       read_bytes_mib, decoded_bytes_mib, elapsed, read_speed, decode_speed, error_count));
+
+    if errors.load(Ordering::SeqCst) > 0 {
         bail!("chunks could not be verified");
     }
 
@@ -106,12 +203,12 @@ fn verify_index_chunks(
 }
 
 fn verify_fixed_index(
-    datastore: &DataStore,
+    datastore: Arc<DataStore>,
     backup_dir: &BackupDir,
     info: &FileInfo,
-    verified_chunks: &mut HashSet<[u8;32]>,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
-    corrupt_chunks: &mut HashSet<[u8;32]>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
-    worker: &WorkerTask,
+    worker: Arc<WorkerTask>,
 ) -> Result<(), Error> {
 
     let mut path = backup_dir.relative_path();
@@ -132,12 +229,12 @@ fn verify_fixed_index(
 }
 
 fn verify_dynamic_index(
-    datastore: &DataStore,
+    datastore: Arc<DataStore>,
     backup_dir: &BackupDir,
     info: &FileInfo,
-    verified_chunks: &mut HashSet<[u8;32]>,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
-    corrupt_chunks: &mut HashSet<[u8;32]>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
-    worker: &WorkerTask,
+    worker: Arc<WorkerTask>,
 ) -> Result<(), Error> {
 
     let mut path = backup_dir.relative_path();
@@ -167,14 +264,14 @@ fn verify_dynamic_index(
 /// - Ok(false) if there were verification errors
 /// - Err(_) if task was aborted
 pub fn verify_backup_dir(
-    datastore: &DataStore,
+    datastore: Arc<DataStore>,
     backup_dir: &BackupDir,
-    verified_chunks: &mut HashSet<[u8;32]>,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
-    corrupt_chunks: &mut HashSet<[u8;32]>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
-    worker: &WorkerTask
+    worker: Arc<WorkerTask>
 ) -> Result<bool, Error> {
 
-    let manifest = match datastore.load_manifest(&backup_dir) {
+    let mut manifest = match datastore.load_manifest(&backup_dir) {
         Ok((manifest, _)) => manifest,
         Err(err) => {
             worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err));
@@ -186,40 +283,53 @@ pub fn verify_backup_dir(
 
     let mut error_count = 0;
 
+    let mut verify_result = "ok";
     for info in manifest.files() {
         let result = proxmox::try_block!({
             worker.log(format!("  check {}", info.filename));
             match archive_type(&info.filename)? {
                 ArchiveType::FixedIndex =>
                     verify_fixed_index(
-                        &datastore,
+                        datastore.clone(),
                         &backup_dir,
                         info,
-                        verified_chunks,
+                        verified_chunks.clone(),
-                        corrupt_chunks,
+                        corrupt_chunks.clone(),
-                        worker
+                        worker.clone(),
                     ),
                 ArchiveType::DynamicIndex =>
                     verify_dynamic_index(
-                        &datastore,
+                        datastore.clone(),
                         &backup_dir,
                         info,
-                        verified_chunks,
+                        verified_chunks.clone(),
-                        corrupt_chunks,
+                        corrupt_chunks.clone(),
-                        worker
+                        worker.clone(),
                     ),
-                ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info),
+                ArchiveType::Blob => verify_blob(datastore.clone(), &backup_dir, info),
             }
         });
 
         worker.fail_on_abort()?;
+        crate::tools::fail_on_shutdown()?;
 
         if let Err(err) = result {
             worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err));
             error_count += 1;
+            verify_result = "failed";
         }
+
     }
 
+    let verify_state = SnapshotVerifyState {
+        state: verify_result.to_string(),
+        upid: worker.upid().clone(),
+    };
+    manifest.unprotected["verify_state"] = serde_json::to_value(verify_state)?;
+    datastore.store_manifest(&backup_dir, serde_json::to_value(manifest)?)
+        .map_err(|err| format_err!("unable to store manifest blob - {}", err))?;
+
+
     Ok(error_count == 0)
 }
 
@@ -228,32 +338,45 @@ pub fn verify_backup_dir(
 /// Errors are logged to the worker log.
 ///
 /// Returns
-/// - Ok(failed_dirs) where failed_dirs had verification errors
+/// - Ok((count, failed_dirs)) where failed_dirs had verification errors
 /// - Err(_) if task was aborted
-pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<Vec<String>, Error> {
+pub fn verify_backup_group(
+    datastore: Arc<DataStore>,
+    group: &BackupGroup,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    progress: Option<(usize, usize)>, // (done, snapshot_count)
+    worker: Arc<WorkerTask>,
+) -> Result<(usize, Vec<String>), Error> {
 
     let mut errors = Vec::new();
     let mut list = match group.list_backups(&datastore.base_path()) {
         Ok(list) => list,
         Err(err) => {
             worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err));
-            return Ok(errors);
+            return Ok((0, errors));
         }
     };
 
     worker.log(format!("verify group {}:{}", datastore.name(), group));
 
-    let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB)
+    let (done, snapshot_count) = progress.unwrap_or((0, list.len()));
-    let mut corrupt_chunks = HashSet::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones
 
+    let mut count = 0;
     BackupInfo::sort_list(&mut list, false); // newest first
     for info in list {
-        if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, &mut corrupt_chunks, worker)?{
+        count += 1;
+        if !verify_backup_dir(datastore.clone(), &info.backup_dir, verified_chunks.clone(), corrupt_chunks.clone(), worker.clone())?{
             errors.push(info.backup_dir.to_string());
         }
+        if snapshot_count != 0 {
+            let pos = done + count;
+            let percentage = ((pos as f64) * 100.0)/(snapshot_count as f64);
+            worker.log(format!("percentage done: {:.2}% ({} of {} snapshots)", percentage, pos, snapshot_count));
+        }
     }
 
-    Ok(errors)
+    Ok((count, errors))
 }
 
 /// Verify all backups inside a datastore
@@ -263,23 +386,49 @@ pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &
 /// Returns
 /// - Ok(failed_dirs) where failed_dirs had verification errors
 /// - Err(_) if task was aborted
-pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<Vec<String>, Error> {
+pub fn verify_all_backups(datastore: Arc<DataStore>, worker: Arc<WorkerTask>) -> Result<Vec<String>, Error> {
 
     let mut errors = Vec::new();
 
-    let list = match BackupGroup::list_groups(&datastore.base_path()) {
+    let mut list = match BackupGroup::list_groups(&datastore.base_path()) {
-        Ok(list) => list,
+        Ok(list) => list
+            .into_iter()
+            .filter(|group| !(group.backup_type() == "host" && group.backup_id() == "benchmark"))
+            .collect::<Vec<BackupGroup>>(),
         Err(err) => {
             worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
             return Ok(errors);
         }
     };
 
-    worker.log(format!("verify datastore {}", datastore.name()));
+    list.sort_unstable();
 
+    let mut snapshot_count = 0;
+    for group in list.iter() {
+        snapshot_count += group.list_backups(&datastore.base_path())?.len();
+    }
+
+    // start with 16384 chunks (up to 65GB)
+    let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*16)));
+
+    // start with 64 chunks since we assume there are few corrupt ones
+    let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
+
+    worker.log(format!("verify datastore {} ({} snapshots)", datastore.name(), snapshot_count));
+
+    let mut done = 0;
     for group in list {
-        let mut group_errors = verify_backup_group(datastore, &group, worker)?;
+        let (count, mut group_errors) = verify_backup_group(
+            datastore.clone(),
+            &group,
+            verified_chunks.clone(),
+            corrupt_chunks.clone(),
+            Some((done, snapshot_count)),
+            worker.clone(),
+        )?;
         errors.append(&mut group_errors);
+
+        done += count;
     }
 
     Ok(errors)

src/bin/proxmox-backup-client.rs

@@ -1026,6 +1026,7 @@ async fn create_backup(
         &backup_id,
         backup_time,
         verbose,
+        false
     ).await?;
 
     let previous_manifest = if let Ok(previous_manifest) = client.download_previous_manifest().await {

src/bin/proxmox-backup-proxy.rs

@@ -20,13 +20,19 @@ use proxmox_backup::tools::disks::{ DiskManage, zfs_pool_stats };
 
 use proxmox_backup::api2::pull::do_sync_job;
 
-fn main() {
+fn main() -> Result<(), Error> {
     proxmox_backup::tools::setup_safe_path_env();
 
-    if let Err(err) = proxmox_backup::tools::runtime::main(run()) {
+    let backup_uid = proxmox_backup::backup::backup_user()?.uid;
-        eprintln!("Error: {}", err);
+    let backup_gid = proxmox_backup::backup::backup_group()?.gid;
-        std::process::exit(-1);
+    let running_uid = nix::unistd::Uid::effective();
+    let running_gid = nix::unistd::Gid::effective();
+
+    if running_uid != backup_uid || running_gid != backup_gid {
+        bail!("proxy not running as backup user or group (got uid {} gid {})", running_uid, running_gid);
     }
+
+    proxmox_backup::tools::runtime::main(run())
 }
 
 async fn run() -> Result<(), Error> {
@@ -43,15 +49,11 @@ async fn run() -> Result<(), Error> {
     let mut config = ApiConfig::new(
         buildcfg::JS_DIR, &proxmox_backup::api2::ROUTER, RpcEnvironmentType::PUBLIC)?;
 
-    // add default dirs which includes jquery and bootstrap
-    // my $base = '/usr/share/libpve-http-server-perl';
-    // add_dirs($self->{dirs}, '/css/' => "$base/css/");
-    // add_dirs($self->{dirs}, '/js/' => "$base/js/");
-    // add_dirs($self->{dirs}, '/fonts/' => "$base/fonts/");
     config.add_alias("novnc", "/usr/share/novnc-pve");
     config.add_alias("extjs", "/usr/share/javascript/extjs");
     config.add_alias("fontawesome", "/usr/share/fonts-font-awesome");
     config.add_alias("xtermjs", "/usr/share/pve-xtermjs");
+    config.add_alias("locale", "/usr/share/pbs-i18n");
     config.add_alias("widgettoolkit", "/usr/share/javascript/proxmox-widget-toolkit");
     config.add_alias("css", "/usr/share/javascript/proxmox-backup/css");
     config.add_alias("docs", "/usr/share/doc/proxmox-backup/html");
@@ -85,8 +87,6 @@ async fn run() -> Result<(), Error> {
                     let acceptor = Arc::clone(&acceptor);
                     async move {
                         sock.set_nodelay(true).unwrap();
-                        sock.set_send_buffer_size(1024*1024).unwrap();
-                        sock.set_recv_buffer_size(1024*1024).unwrap();
                         Ok(tokio_openssl::accept(&acceptor, sock)
                             .await
                             .ok() // handshake errors aren't be fatal, so return None to filter
@@ -300,7 +300,8 @@ async fn schedule_datastore_garbage_collection() {
         };
 
         let next = match compute_next_event(&event, last, false) {
-            Ok(next) => next,
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
             Err(err) => {
                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
                 continue;
@@ -411,7 +412,8 @@ async fn schedule_datastore_prune() {
         };
 
         let next = match compute_next_event(&event, last, false) {
-            Ok(next) => next,
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
             Err(err) => {
                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
                 continue;
@@ -519,7 +521,8 @@ async fn schedule_datastore_sync_jobs() {
         };
 
         let next = match compute_next_event(&event, last, false) {
-            Ok(next) => next,
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
             Err(err) => {
                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
                 continue;

src/bin/proxmox_backup_client/benchmark.rs

@@ -226,6 +226,7 @@ async fn test_upload_speed(
         "benchmark",
         backup_time,
         false,
+        true
     ).await?;
 
     if verbose { eprintln!("Start TLS speed test"); }

src/bin/proxmox_backup_client/mount.rs

@@ -141,7 +141,7 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
 
     let (manifest, _) = client.download_manifest().await?;
 
-    let file_info = manifest.lookup_file_info(&archive_name)?;
+    let file_info = manifest.lookup_file_info(&server_archive_name)?;
 
     if server_archive_name.ends_with(".didx") {
         let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;

src/bin/proxmox_backup_manager/disk.rs

@@ -239,7 +239,7 @@ pub fn zpool_commands() -> CommandLineInterface {
         .insert("create",
                 CliCommand::new(&API_METHOD_CREATE_ZPOOL)
                 .arg_param(&["name"])
-                .completion_cb("devices", complete_disk_name) // fixme: comlete the list
+                .completion_cb("devices", complete_disk_name) // fixme: complete the list
         );
 
     cmd_def.into()

src/client/backup_writer.rs

@@ -53,6 +53,7 @@ impl BackupWriter {
         backup_id: &str,
         backup_time: DateTime<Utc>,
         debug: bool,
+        benchmark: bool
     ) -> Result<Arc<BackupWriter>, Error> {
 
         let param = json!({
@@ -60,7 +61,8 @@ impl BackupWriter {
             "backup-id": backup_id,
             "backup-time": backup_time.timestamp(),
             "store": datastore,
-            "debug": debug
+            "debug": debug,
+            "benchmark": benchmark
         });
 
         let req = HttpClient::request_builder(
@@ -629,7 +631,7 @@ impl BackupWriter {
             })
     }
 
-    /// Upload speed test - prints result ot stderr
+    /// Upload speed test - prints result to stderr
     pub async fn upload_speedtest(&self, verbose: bool) -> Result<f64, Error> {
 
         let mut data = vec![];

src/client/http_client.rs

@@ -292,7 +292,6 @@ impl HttpClient {
 
         let mut httpc = hyper::client::HttpConnector::new();
         httpc.set_nodelay(true); // important for h2 download performance!
-        httpc.set_recv_buffer_size(Some(1024*1024)); //important for h2 download performance!
         httpc.enforce_http(false); // we want https...
 
         let https = HttpsConnector::with_connector(httpc, ssl_connector_builder.build());

src/server/rest.rs

@@ -313,7 +313,13 @@ pub async fn handle_api_request<Env: RpcEnvironment, S: 'static + BuildHasher +
     Ok(resp)
 }
 
-fn get_index(userid: Option<Userid>, token: Option<String>, api: &Arc<ApiConfig>, parts: Parts) ->  Response<Body> {
+fn get_index(
+    userid: Option<Userid>,
+    token: Option<String>,
+    language: Option<String>,
+    api: &Arc<ApiConfig>,
+    parts: Parts,
+) ->  Response<Body> {
 
     let nodename = proxmox::tools::nodename();
     let userid = userid.as_ref().map(|u| u.as_str()).unwrap_or("");
@@ -333,10 +339,18 @@ fn get_index(userid: Option<Userid>, token: Option<String>, api: &Arc<ApiConfig>
         }
     }
 
+    let mut lang = String::from("");
+    if let Some(language) = language {
+        if Path::new(&format!("/usr/share/pbs-i18n/pbs-lang-{}.js", language)).exists() {
+            lang = language;
+        }
+    }
+
     let data = json!({
         "NodeName": nodename,
         "UserName": userid,
         "CSRFPreventionToken": token,
+        "language": lang,
         "debug": debug,
     });
 
@@ -441,12 +455,14 @@ async fn handle_static_file_download(filename: PathBuf) ->  Result<Response<Body
     }
 }
 
-fn extract_auth_data(headers: &http::HeaderMap) -> (Option<String>, Option<String>) {
+fn extract_auth_data(headers: &http::HeaderMap) -> (Option<String>, Option<String>, Option<String>) {
 
     let mut ticket = None;
+    let mut language = None;
     if let Some(raw_cookie) = headers.get("COOKIE") {
         if let Ok(cookie) = raw_cookie.to_str() {
-            ticket = tools::extract_auth_cookie(cookie, "PBSAuthCookie");
+            ticket = tools::extract_cookie(cookie, "PBSAuthCookie");
+            language = tools::extract_cookie(cookie, "PBSLangCookie");
         }
     }
 
@@ -455,7 +471,7 @@ fn extract_auth_data(headers: &http::HeaderMap) -> (Option<String>, Option<Strin
         _ => None,
     };
 
-    (ticket, token)
+    (ticket, token, language)
 }
 
 fn check_auth(
@@ -526,7 +542,7 @@ pub async fn handle_request(api: Arc<ApiConfig>, req: Request<Body>) -> Result<R
             ) {
                 // explicitly allow those calls without auth
             } else {
-                let (ticket, token) = extract_auth_data(&parts.headers);
+                let (ticket, token, _) = extract_auth_data(&parts.headers);
                 match check_auth(&method, &ticket, &token, &user_info) {
                     Ok(userid) => rpcenv.set_user(Some(userid.to_string())),
                     Err(err) => {
@@ -573,20 +589,20 @@ pub async fn handle_request(api: Arc<ApiConfig>, req: Request<Body>) -> Result<R
         }
 
         if comp_len == 0 {
-            let (ticket, token) = extract_auth_data(&parts.headers);
+            let (ticket, token, language) = extract_auth_data(&parts.headers);
             if ticket != None {
                 match check_auth(&method, &ticket, &token, &user_info) {
                     Ok(userid) => {
                         let new_token = assemble_csrf_prevention_token(csrf_secret(), &userid);
-                        return Ok(get_index(Some(userid), Some(new_token), &api, parts));
+                        return Ok(get_index(Some(userid), Some(new_token), language, &api, parts));
                     }
                     _ => {
                         tokio::time::delay_until(Instant::from_std(delay_unauth_time)).await;
-                        return Ok(get_index(None, None, &api, parts));
+                        return Ok(get_index(None, None, language, &api, parts));
                     }
                 }
             } else {
-                return Ok(get_index(None, None, &api, parts));
+                return Ok(get_index(None, None, language, &api, parts));
             }
         } else {
             let filename = api.find_alias(&components);

src/server/worker_task.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 use std::fs::File;
-use std::io::{BufRead, BufReader};
+use std::io::{Read, BufRead, BufReader};
 use std::panic::UnwindSafe;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
@@ -195,8 +195,8 @@ pub fn create_task_log_dirs() -> Result<(), Error> {
 /// If there is not a single line with at valid datetime, we assume the
 /// starttime to be the endtime
 pub fn upid_read_status(upid: &UPID) -> Result<TaskState, Error> {
-    let mut endtime = upid.starttime;
+
-    let mut status = TaskState::Unknown { endtime };
+    let mut status = TaskState::Unknown { endtime: upid.starttime };
 
     let path = upid.log_path();
 
@@ -207,22 +207,34 @@ pub fn upid_read_status(upid: &UPID) -> Result<TaskState, Error> {
     use std::io::SeekFrom;
     let _ = file.seek(SeekFrom::End(-8192)); // ignore errors
 
-    let reader = BufReader::new(file);
+    let mut data = Vec::with_capacity(8192);
+    file.read_to_end(&mut data)?;
 
-    for line in reader.lines() {
+    // task logs should end with newline, we do not want it here
-        let line = line?;
+    if data[data.len()-1] == b'\n' {
+        data.pop();
+    }
 
-        let mut iter = line.splitn(2, ": ");
+    let last_line = {
-        if let Some(time_str) = iter.next() {
+        let mut start = 0;
-            endtime = chrono::DateTime::parse_from_rfc3339(time_str)
+        for pos in (0..data.len()).rev() {
-                .map_err(|err| format_err!("cannot parse '{}': {}", time_str, err))?
+            if data[pos] == b'\n' {
-                .timestamp();
+                start = pos + 1;
-        } else {
+                break;
-            continue;
+            }
         }
-        match iter.next().and_then(|rest| rest.strip_prefix("TASK ")) {
+        &data[start..]
-            None => continue,
+    };
-            Some(rest) => {
+
+    let last_line = std::str::from_utf8(last_line)
+        .map_err(|err| format_err!("upid_read_status: utf8 parse failed: {}", err))?;
+
+    let mut iter = last_line.splitn(2, ": ");
+    if let Some(time_str) = iter.next() {
+        if let Ok(endtime) = chrono::DateTime::parse_from_rfc3339(time_str) {
+            let endtime = endtime.timestamp();
+
+            if let Some(rest) = iter.next().and_then(|rest| rest.strip_prefix("TASK ")) {
                 if let Ok(state) = TaskState::from_endtime_and_message(endtime, rest) {
                     status = state;
                 }

src/tools.rs

@@ -326,9 +326,9 @@ pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
     Ok(())
 }
 
-/// Extract authentication cookie from cookie header.
+/// Extract a specific cookie from cookie header.
 /// We assume cookie_name is already url encoded.
-pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
+pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
     for pair in cookie.split(';') {
         let (name, value) = match pair.find('=') {
             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),

src/tools/disks.rs

@@ -133,7 +133,7 @@ impl DiskManage {
             })
     }
 
-    /// Information about file system type and unsed device for a path
+    /// Information about file system type and used device for a path
     ///
     /// Returns tuple (fs_type, device, mount_source)
     pub fn find_mounted_device(

src/tools/disks/zpool_list.rs

@@ -111,7 +111,7 @@ fn parse_zpool_list_item(i: &str) -> IResult<&str, ZFSPoolInfo> {
     Ok((i, stat))
 }
 
-/// Parse zpool list outout
+/// Parse zpool list output
 ///
 /// Note: This does not reveal any details on how the pool uses the devices, because
 /// the zpool list output format is not really defined...

src/tools/disks/zpool_status.rs

@@ -53,7 +53,7 @@ fn parse_zpool_status_vdev(i: &str) -> IResult<&str, ZFSPoolVDevState> {
 
     let (i, vdev_name) =  notspace1(i)?;
 
-    if let Ok((n, _)) = preceded(multispace0, line_ending)(i) { // sepecial device
+    if let Ok((n, _)) = preceded(multispace0, line_ending)(i) { // special device
         let vdev = ZFSPoolVDevState {
             name: vdev_name.to_string(),
             lvl: indent_level,

src/tools/format.rs

@@ -80,6 +80,11 @@ impl From<usize> for HumanByte {
         HumanByte { b: v }
     }
 }
+impl From<u64> for HumanByte {
+    fn from(v: u64) -> Self {
+        HumanByte { b: v as usize }
+    }
+}
 
 #[test]
 fn correct_byte_convert() {

src/tools/systemd/parse_time.rs

@@ -145,6 +145,9 @@ fn parse_date_time_comp(max: usize) -> impl Fn(&str) -> IResult<&str, DateTimeVa
         let (i, value) = parse_time_comp(max)(i)?;
 
         if let (i, Some(end)) = opt(preceded(tag(".."), parse_time_comp(max)))(i)? {
+            if value > end {
+                return Err(parse_error(i, "range start is bigger than end"));
+            }
             return Ok((i, DateTimeValue::Range(value, end)))
         }
 
@@ -183,6 +186,25 @@ fn parse_time_spec(i: &str) -> IResult<&str, (Vec<DateTimeValue>, Vec<DateTimeVa
     }
 }
 
+fn parse_date_spec(i: &str) -> IResult<&str, (Vec<DateTimeValue>, Vec<DateTimeValue>, Vec<DateTimeValue>)> {
+
+    // TODO: implement ~ for days (man systemd.time)
+    if let Ok((i, (year, month, day))) = tuple((
+        parse_date_time_comp_list(2200), // the upper limit for systemd, stay compatible
+        preceded(tag("-"), parse_date_time_comp_list(13)),
+        preceded(tag("-"), parse_date_time_comp_list(32)),
+    ))(i) {
+        Ok((i, (year, month, day)))
+    } else if let Ok((i, (month, day))) = tuple((
+        parse_date_time_comp_list(13),
+        preceded(tag("-"), parse_date_time_comp_list(32)),
+    ))(i) {
+        Ok((i, (Vec::new(), month, day)))
+    } else {
+        Err(parse_error(i, "invalid date spec"))
+    }
+}
+
 pub fn parse_calendar_event(i: &str) -> Result<CalendarEvent, Error> {
     parse_complete_line("calendar event", i, parse_calendar_event_incomplete)
 }
@@ -191,7 +213,7 @@ fn parse_calendar_event_incomplete(mut i: &str) -> IResult<&str, CalendarEvent>
 
     let mut has_dayspec = false;
     let mut has_timespec = false;
-    let has_datespec = false;
+    let mut has_datespec = false;
 
     let mut event = CalendarEvent::default();
 
@@ -228,8 +250,52 @@ fn parse_calendar_event_incomplete(mut i: &str) -> IResult<&str, CalendarEvent>
                     ..Default::default()
                 }));
             }
-            "monthly" | "yearly" | "quarterly" | "semiannually" => {
+            "monthly" => {
-                return Err(parse_error(i, "unimplemented date or time specification"));
+                return Ok(("", CalendarEvent {
+                    hour: vec![DateTimeValue::Single(0)],
+                    minute: vec![DateTimeValue::Single(0)],
+                    second: vec![DateTimeValue::Single(0)],
+                    day: vec![DateTimeValue::Single(1)],
+                    ..Default::default()
+                }));
+            }
+            "yearly" | "annually" => {
+                return Ok(("", CalendarEvent {
+                    hour: vec![DateTimeValue::Single(0)],
+                    minute: vec![DateTimeValue::Single(0)],
+                    second: vec![DateTimeValue::Single(0)],
+                    day: vec![DateTimeValue::Single(1)],
+                    month: vec![DateTimeValue::Single(1)],
+                    ..Default::default()
+                }));
+            }
+            "quarterly" => {
+                return Ok(("", CalendarEvent {
+                    hour: vec![DateTimeValue::Single(0)],
+                    minute: vec![DateTimeValue::Single(0)],
+                    second: vec![DateTimeValue::Single(0)],
+                    day: vec![DateTimeValue::Single(1)],
+                    month: vec![
+                        DateTimeValue::Single(1),
+                        DateTimeValue::Single(4),
+                        DateTimeValue::Single(7),
+                        DateTimeValue::Single(10),
+                    ],
+                    ..Default::default()
+                }));
+            }
+            "semiannually" | "semi-annually" => {
+                return Ok(("", CalendarEvent {
+                    hour: vec![DateTimeValue::Single(0)],
+                    minute: vec![DateTimeValue::Single(0)],
+                    second: vec![DateTimeValue::Single(0)],
+                    day: vec![DateTimeValue::Single(1)],
+                    month: vec![
+                        DateTimeValue::Single(1),
+                        DateTimeValue::Single(7),
+                    ],
+                    ..Default::default()
+                }));
             }
             _ => { /* continue */ }
         }
@@ -246,7 +312,13 @@ fn parse_calendar_event_incomplete(mut i: &str) -> IResult<&str, CalendarEvent>
         for range in range_list  { event.days.insert(range); }
     }
 
-    // todo: support date specs
+    if let (n, Some((year, month, day))) = opt(parse_date_spec)(i)? {
+        event.year = year;
+        event.month = month;
+        event.day = day;
+        has_datespec = true;
+        i = space0(n)?.0;
+    }
 
     if let (n, Some((hour, minute, second))) = opt(parse_time_spec)(i)? {
         event.hour = hour;

src/tools/systemd/time.rs

@@ -1,4 +1,6 @@
-use anyhow::{bail, Error};
+use std::convert::TryInto;
+
+use anyhow::Error;
 use bitflags::bitflags;
 
 pub use super::parse_time::*;
@@ -17,7 +19,7 @@ bitflags!{
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum DateTimeValue {
     Single(u32),
     Range(u32, u32),
@@ -54,7 +56,7 @@ impl DateTimeValue {
         let mut next: Option<u32> = None;
         let mut set_next = |v: u32| {
             if let Some(n) = next {
-                if v > n { next = Some(v); }
+                if v < n { next = Some(v); }
             } else {
                 next = Some(v);
             }
@@ -91,7 +93,7 @@ impl DateTimeValue {
 /// Calendar events may be used to refer to one or more points in time in a
 /// single expression. They are designed after the systemd.time Calendar Events
 /// specification, but are not guaranteed to be 100% compatible.
-#[derive(Default, Debug)]
+#[derive(Default, Clone, Debug)]
 pub struct CalendarEvent {
     /// the days in a week this event should trigger
     pub days: WeekDays,
@@ -101,17 +103,15 @@ pub struct CalendarEvent {
     pub minute: Vec<DateTimeValue>,
     /// the hour(s) this event should trigger
     pub hour: Vec<DateTimeValue>,
-/* FIXME: TODO
     /// the day(s) in a month this event should trigger
     pub day: Vec<DateTimeValue>,
     /// the month(s) in a year this event should trigger
     pub month: Vec<DateTimeValue>,
     /// the years(s) this event should trigger
     pub year: Vec<DateTimeValue>,
-*/
 }
 
-#[derive(Default)]
+#[derive(Default, Clone, Debug)]
 pub struct TimeSpan {
     pub nsec: u64,
     pub usec: u64,
@@ -155,7 +155,7 @@ pub fn compute_next_event(
     event: &CalendarEvent,
     last: i64,
     utc: bool,
-) -> Result<i64, Error> {
+) -> Result<Option<i64>, Error> {
 
     let last = last + 1; // at least one second later
 
@@ -166,94 +166,124 @@ pub fn compute_next_event(
     let mut count = 0;
 
     loop {
-        if count > 1000 { // should not happen
+        // cancel after 1000 loops
-            bail!("unable to compute next calendar event");
+        if count > 1000 {
+            return Ok(None);
         } else {
             count += 1;
         }
 
-        if !all_days && t.changes.contains(TMChanges::WDAY) { // match day first
+        if !event.year.is_empty() {
-            let day_num = t.day_num();
+            let year: u32 = t.year().try_into()?;
+            if !DateTimeValue::list_contains(&event.year, year) {
+                if let Some(n) = DateTimeValue::find_next(&event.year, year) {
+                    t.add_years((n - year).try_into()?)?;
+                    continue;
+                } else {
+                    // if we have no valid year, we cannot find a correct timestamp
+                    return Ok(None);
+                }
+            }
+        }
+
+        if !event.month.is_empty() {
+            let month: u32 = t.month().try_into()?;
+            if !DateTimeValue::list_contains(&event.month, month) {
+                if let Some(n) = DateTimeValue::find_next(&event.month, month) {
+                    t.add_months((n - month).try_into()?)?;
+                } else {
+                    // if we could not find valid month, retry next year
+                    t.add_years(1)?;
+                }
+                continue;
+            }
+        }
+
+        if !event.day.is_empty() {
+            let day: u32 = t.day().try_into()?;
+            if !DateTimeValue::list_contains(&event.day, day) {
+                if let Some(n) = DateTimeValue::find_next(&event.day, day) {
+                    t.add_days((n - day).try_into()?)?;
+                } else {
+                    // if we could not find valid mday, retry next month
+                    t.add_months(1)?;
+                }
+                continue;
+            }
+        }
+
+        if !all_days { // match day first
+            let day_num: u32 = t.day_num().try_into()?;
             let day = WeekDays::from_bits(1<<day_num).unwrap();
-            if event.days.contains(day) {
+            if !event.days.contains(day) {
-                t.changes.remove(TMChanges::WDAY);
-            } else {
                 if let Some(n) = ((day_num+1)..7)
                     .find(|d| event.days.contains(WeekDays::from_bits(1<<d).unwrap()))
                 {
                     // try next day
-                    t.add_days(n - day_num, true);
+                    t.add_days((n - day_num).try_into()?)?;
-                    continue;
                 } else {
                     // try next week
-                    t.add_days(7 - day_num, true);
+                    t.add_days((7 - day_num).try_into()?)?;
-                    continue;
                 }
+                continue;
             }
         }
 
         // this day
-        if !event.hour.is_empty() && t.changes.contains(TMChanges::HOUR) {
+        if !event.hour.is_empty() {
-            let hour = t.hour() as u32;
+            let hour = t.hour().try_into()?;
-            if DateTimeValue::list_contains(&event.hour, hour) {
+            if !DateTimeValue::list_contains(&event.hour, hour) {
-                t.changes.remove(TMChanges::HOUR);
-            } else {
                 if let Some(n) = DateTimeValue::find_next(&event.hour, hour) {
                     // test next hour
-                    t.set_time(n as libc::c_int, 0, 0);
+                    t.set_time(n.try_into()?, 0, 0)?;
-                    continue;
                 } else {
                     // test next day
-                    t.add_days(1, true);
+                    t.add_days(1)?;
-                    continue;
                 }
+                continue;
             }
         }
 
         // this hour
-        if !event.minute.is_empty()  && t.changes.contains(TMChanges::MIN) {
+        if !event.minute.is_empty() {
-            let minute = t.min() as u32;
+            let minute = t.min().try_into()?;
-            if DateTimeValue::list_contains(&event.minute, minute) {
+            if !DateTimeValue::list_contains(&event.minute, minute) {
-                t.changes.remove(TMChanges::MIN);
-            } else {
                 if let Some(n) = DateTimeValue::find_next(&event.minute, minute) {
                     // test next minute
-                    t.set_min_sec(n as libc::c_int, 0);
+                    t.set_min_sec(n.try_into()?, 0)?;
-                    continue;
                 } else {
                     // test next hour
-                    t.set_time(t.hour() + 1, 0, 0);
+                    t.set_time(t.hour() + 1, 0, 0)?;
-                    continue;
                 }
+                continue;
             }
         }
 
         // this minute
-        if !event.second.is_empty() && t.changes.contains(TMChanges::SEC) {
+        if !event.second.is_empty() {
-            let second = t.sec() as u32;
+            let second = t.sec().try_into()?;
-            if DateTimeValue::list_contains(&event.second, second) {
+            if !DateTimeValue::list_contains(&event.second, second) {
-                t.changes.remove(TMChanges::SEC);
-            } else {
                 if let Some(n) = DateTimeValue::find_next(&event.second, second) {
                     // test next second
-                    t.set_sec(n as libc::c_int);
+                    t.set_sec(n.try_into()?)?;
-                    continue;
                 } else {
                     // test next min
-                    t.set_min_sec(t.min() + 1, 0);
+                    t.set_min_sec(t.min() + 1, 0)?;
-                    continue;
                 }
+                continue;
             }
         }
 
         let next = t.into_epoch()?;
-        return Ok(next)
+        return Ok(Some(next))
     }
 }
 
 #[cfg(test)]
 mod test {
 
+    use anyhow::bail;
+
     use super::*;
     use proxmox::tools::time::*;
 
@@ -280,7 +310,7 @@ mod test {
             };
 
             match compute_next_event(&event, last, true) {
-                Ok(next) => {
+                Ok(Some(next)) => {
                     if next == expect {
                         println!("next {:?} => {}", event, next);
                     } else {
@@ -288,12 +318,25 @@ mod test {
                               event, gmtime(next), gmtime(expect));
                     }
                 }
+                Ok(None) => bail!("next {:?} failed to find a timestamp", event),
                 Err(err) => bail!("compute next for '{}' failed - {}", v, err),
             }
 
             Ok(expect)
         };
 
+        let test_never = |v: &'static str, last: i64| -> Result<(), Error> {
+            let event = match parse_calendar_event(v) {
+                Ok(event) => event,
+                Err(err) => bail!("parsing '{}' failed - {}", v, err),
+            };
+
+            match compute_next_event(&event, last, true)? {
+                None => Ok(()),
+                Some(next) => bail!("compute next for '{}' succeeded, but expected fail - result {}", v, next),
+            }
+        };
+
         const MIN: i64 = 60;
         const HOUR: i64 = 3600;
         const DAY: i64 = 3600*24;
@@ -320,6 +363,13 @@ mod test {
         test_value("sat", THURSDAY_00_00, THURSDAY_00_00 + 2*DAY)?;
         test_value("sun", THURSDAY_00_00, THURSDAY_00_00 + 3*DAY)?;
 
+        // test multiple values for a single field
+        // and test that the order does not matter
+        test_value("5,10:4,8", THURSDAY_00_00, THURSDAY_00_00 + 5*HOUR + 4*MIN)?;
+        test_value("10,5:8,4", THURSDAY_00_00, THURSDAY_00_00 + 5*HOUR + 4*MIN)?;
+        test_value("6,4..10:23,5/5", THURSDAY_00_00, THURSDAY_00_00 + 4*HOUR + 5*MIN)?;
+        test_value("4..10,6:5/5,23", THURSDAY_00_00, THURSDAY_00_00 + 4*HOUR + 5*MIN)?;
+
         // test month wrapping
         test_value("sat", JUL_31_2020, JUL_31_2020 + 1*DAY)?;
         test_value("sun", JUL_31_2020, JUL_31_2020 + 2*DAY)?;
@@ -361,6 +411,23 @@ mod test {
             n = test_value("1:0", n, THURSDAY_00_00 + i*DAY + HOUR)?;
         }
 
+        // test date functionality
+
+        test_value("2020-07-31", 0, JUL_31_2020)?;
+        test_value("02-28", 0, (31+27)*DAY)?;
+        test_value("02-29", 0, 2*365*DAY + (31+28)*DAY)?; // 1972-02-29
+        test_value("1965/5-01-01", -1, THURSDAY_00_00)?;
+        test_value("2020-7..9-2/2", JUL_31_2020, JUL_31_2020 + 2*DAY)?;
+        test_value("2020,2021-12-31", JUL_31_2020, DEC_31_2020)?;
+
+        test_value("monthly", 0, 31*DAY)?;
+        test_value("quarterly", 0, (31+28+31)*DAY)?;
+        test_value("semiannually", 0, (31+28+31+30+31+30)*DAY)?;
+        test_value("yearly", 0, (365)*DAY)?;
+
+        test_never("2021-02-29", 0)?;
+        test_never("02-30", 0)?;
+
         Ok(())
     }
 

src/tools/systemd/tm_editor.rs

@@ -1,73 +1,60 @@
 use anyhow::Error;
-use bitflags::bitflags;
 
 use proxmox::tools::time::*;
 
-bitflags!{
-    #[derive(Default)]
-    pub struct TMChanges: u8 {
-        const SEC = 1;
-        const MIN = 2;
-        const HOUR = 4;
-        const MDAY = 8;
-        const MON = 16;
-        const YEAR = 32;
-        const WDAY = 64;
-    }
-}
-
 pub struct TmEditor {
     utc: bool,
     t: libc::tm,
-    pub changes: TMChanges,
-}
-
-fn is_leap_year(year: libc::c_int) -> bool {
-    if year % 4 != 0  { return false; }
-    if year % 100 != 0 { return true; }
-    if year % 400 != 0  { return false; }
-    return true;
-}
-
-fn days_in_month(mon: libc::c_int, year: libc::c_int) -> libc::c_int {
-
-    let mon = mon % 12;
-
-    static MAP: &[libc::c_int] = &[31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
-
-    if mon == 1 && is_leap_year(year) { return 29; }
-
-    MAP[mon as usize]
 }
 
 impl TmEditor {
 
     pub fn new(epoch: i64, utc: bool) -> Result<Self, Error> {
-        let mut t = if utc { gmtime(epoch)? } else { localtime(epoch)? };
+        let t = if utc { gmtime(epoch)? } else { localtime(epoch)? };
-        t.tm_year += 1900; // real years for clarity
+        Ok(Self { utc, t })
-        Ok(Self { utc, t, changes: TMChanges::all() })
     }
 
     pub fn into_epoch(mut self) -> Result<i64, Error> {
-        self.t.tm_year -= 1900;
+        let epoch = if self.utc { timegm(&mut self.t)? } else { timelocal(&mut self.t)? };
-        let epoch = if self.utc { timegm(self.t)? } else { timelocal(self.t)? };
         Ok(epoch)
     }
 
-    pub fn add_days(&mut self, days: libc::c_int, reset_time: bool) {
+    /// increases the year by 'years' and resets all smaller fields to their minimum
-        if days == 0 { return; }
+    pub fn add_years(&mut self, years: libc::c_int) -> Result<(), Error> {
-        if reset_time {
+        if years == 0 { return Ok(()); }
-            self.t.tm_hour = 0;
+        self.t.tm_mon = 0;
-            self.t.tm_min = 0;
+        self.t.tm_mday = 1;
-            self.t.tm_sec = 0;
+        self.t.tm_hour = 0;
-            self.changes.insert(TMChanges::HOUR|TMChanges::MIN|TMChanges::SEC);
+        self.t.tm_min = 0;
-        }
+        self.t.tm_sec = 0;
-        self.t.tm_mday += days;
+        self.t.tm_year += years;
-        self.t.tm_wday += days;
+        self.normalize_time()
-        self.changes.insert(TMChanges::MDAY|TMChanges::WDAY);
-        self.wrap_time();
     }
 
+    /// increases the month by 'months' and resets all smaller fields to their minimum
+    pub fn add_months(&mut self, months: libc::c_int) -> Result<(), Error> {
+        if months == 0 { return Ok(()); }
+        self.t.tm_mday = 1;
+        self.t.tm_hour = 0;
+        self.t.tm_min = 0;
+        self.t.tm_sec = 0;
+        self.t.tm_mon += months;
+        self.normalize_time()
+    }
+
+    /// increases the day by 'days' and resets all smaller fields to their minimum
+    pub fn add_days(&mut self, days: libc::c_int) -> Result<(), Error> {
+        if days == 0 { return Ok(()); }
+        self.t.tm_hour = 0;
+        self.t.tm_min = 0;
+        self.t.tm_sec = 0;
+        self.t.tm_mday += days;
+        self.normalize_time()
+    }
+
+    pub fn year(&self) -> libc::c_int { self.t.tm_year + 1900 } // see man mktime
+    pub fn month(&self) -> libc::c_int { self.t.tm_mon + 1 }
+    pub fn day(&self) -> libc::c_int { self.t.tm_mday }
     pub fn hour(&self) -> libc::c_int { self.t.tm_hour }
     pub fn min(&self) -> libc::c_int { self.t.tm_min }
     pub fn sec(&self) -> libc::c_int { self.t.tm_sec }
@@ -77,109 +64,56 @@ impl TmEditor {
         (self.t.tm_wday + 6) % 7
     }
 
-    pub fn set_time(&mut self, hour: libc::c_int, min: libc::c_int, sec: libc::c_int) {
+    pub fn set_time(&mut self, hour: libc::c_int, min: libc::c_int, sec: libc::c_int) -> Result<(), Error> {
         self.t.tm_hour = hour;
         self.t.tm_min = min;
         self.t.tm_sec = sec;
-        self.changes.insert(TMChanges::HOUR|TMChanges::MIN|TMChanges::SEC);
+        self.normalize_time()
-        self.wrap_time();
     }
 
-    pub fn set_min_sec(&mut self, min: libc::c_int, sec: libc::c_int) {
+    pub fn set_min_sec(&mut self, min: libc::c_int, sec: libc::c_int) -> Result<(), Error> {
         self.t.tm_min = min;
         self.t.tm_sec = sec;
-        self.changes.insert(TMChanges::MIN|TMChanges::SEC);
+        self.normalize_time()
-        self.wrap_time();
     }
 
-    fn wrap_time(&mut self) {
+    fn normalize_time(&mut self) -> Result<(), Error> {
-
+        // libc normalizes it for us
-        // sec: 0..59
+        if self.utc {
-        if self.t.tm_sec >= 60 {
+            timegm(&mut self.t)?;
-            self.t.tm_min += self.t.tm_sec / 60;
+        } else {
-            self.t.tm_sec %= 60;
+            timelocal(&mut self.t)?;
-            self.changes.insert(TMChanges::SEC|TMChanges::MIN);
         }
-
+        Ok(())
-        // min: 0..59
-        if self.t.tm_min >= 60 {
-            self.t.tm_hour += self.t.tm_min / 60;
-            self.t.tm_min %= 60;
-            self.changes.insert(TMChanges::MIN|TMChanges::HOUR);
-       }
-
-        // hour: 0..23
-        if self.t.tm_hour >= 24 {
-            self.t.tm_mday += self.t.tm_hour / 24;
-            self.t.tm_wday += self.t.tm_hour / 24;
-            self.t.tm_hour %= 24;
-            self.changes.insert(TMChanges::HOUR|TMChanges::MDAY|TMChanges::WDAY);
-        }
-
-        // Translate to 0..($days_in_mon-1)
-        self.t.tm_mday -= 1;
-        loop {
-	    let days_in_mon = days_in_month(self.t.tm_mon, self.t.tm_year);
-	    if self.t.tm_mday < days_in_mon { break; }
-	    // Wrap one month
-	    self.t.tm_mday -= days_in_mon;
-	    self.t.tm_mon += 1;
-            self.changes.insert(TMChanges::MDAY|TMChanges::WDAY|TMChanges::MON);
-        }
-
-        // Translate back to 1..$days_in_mon
-        self.t.tm_mday += 1;
-
-        // mon: 0..11
-        if self.t.tm_mon >= 12 {
-            self.t.tm_year += self.t.tm_mon / 12;
-            self.t.tm_mon %= 12;
-            self.changes.insert(TMChanges::MON|TMChanges::YEAR);
-        }
-
-        self.t.tm_wday %= 7;
     }
 
-    pub fn set_sec(&mut self, v: libc::c_int) {
+    pub fn set_sec(&mut self, v: libc::c_int) -> Result<(), Error> {
         self.t.tm_sec = v;
-        self.changes.insert(TMChanges::SEC);
+        self.normalize_time()
-        self.wrap_time();
     }
 
-    pub fn set_min(&mut self, v: libc::c_int) {
+    pub fn set_min(&mut self, v: libc::c_int) -> Result<(), Error> {
         self.t.tm_min = v;
-        self.changes.insert(TMChanges::MIN);
+        self.normalize_time()
-        self.wrap_time();
     }
 
-    pub fn set_hour(&mut self, v: libc::c_int) {
+    pub fn set_hour(&mut self, v: libc::c_int) -> Result<(), Error> {
         self.t.tm_hour = v;
-        self.changes.insert(TMChanges::HOUR);
+        self.normalize_time()
-        self.wrap_time();
     }
 
-    pub fn set_mday(&mut self, v: libc::c_int) {
+    pub fn set_mday(&mut self, v: libc::c_int) -> Result<(), Error> {
         self.t.tm_mday = v;
-        self.changes.insert(TMChanges::MDAY);
+        self.normalize_time()
-        self.wrap_time();
     }
 
-    pub fn set_mon(&mut self, v: libc::c_int) {
+    pub fn set_mon(&mut self, v: libc::c_int) -> Result<(), Error> {
-        self.t.tm_mon = v;
+        self.t.tm_mon = v - 1;
-        self.changes.insert(TMChanges::MON);
+        self.normalize_time()
-        self.wrap_time();
     }
 
-    pub fn set_year(&mut self, v: libc::c_int) {
+    pub fn set_year(&mut self, v: libc::c_int) -> Result<(), Error> {
-        self.t.tm_year = v;
+        self.t.tm_year = v - 1900;
-        self.changes.insert(TMChanges::YEAR);
+        self.normalize_time()
-        self.wrap_time();
     }
-
-    pub fn set_wday(&mut self, v: libc::c_int) {
-        self.t.tm_wday = v;
-        self.changes.insert(TMChanges::WDAY);
-        self.wrap_time();
-    }
-
 }

src/tools/systemd/types.rs

@@ -252,6 +252,6 @@ pub const SYSTEMD_TIMESPAN_SCHEMA: Schema = StringSchema::new(
     .schema();
 
 pub const SYSTEMD_CALENDAR_EVENT_SCHEMA: Schema = StringSchema::new(
-    "systemd time span")
+    "systemd calendar event")
     .format(&ApiStringFormat::VerifyFn(super::time::verify_calendar_event))
     .schema();

www/DataStoreContent.js

@@ -6,16 +6,16 @@ Ext.define('pbs-data-store-snapshots', {
 	{
 	    name: 'backup-time',
 	    type: 'date',
-	    dateFormat: 'timestamp'
+	    dateFormat: 'timestamp',
 	},
 	'files',
 	'owner',
-	{ name: 'size', type: 'int', allowNull: true, },
+	'verification',
+	{ name: 'size', type: 'int', allowNull: true },
 	{
 	    name: 'crypt-mode',
 	    type: 'boolean',
 	    calculate: function(data) {
-		let encrypted = 0;
 		let crypt = {
 		    none: 0,
 		    mixed: 0,
@@ -23,25 +23,24 @@ Ext.define('pbs-data-store-snapshots', {
 		    encrypt: 0,
 		    count: 0,
 		};
-		let signed = 0;
 		data.files.forEach(file => {
 		    if (file.filename === 'index.json.blob') return; // is never encrypted
 		    let mode = PBS.Utils.cryptmap.indexOf(file['crypt-mode']);
 		    if (mode !== -1) {
 			crypt[file['crypt-mode']]++;
+			crypt.count++;
 		    }
-		    crypt.count++;
 		});
 
 		return PBS.Utils.calculateCryptMode(crypt);
-	    }
+	    },
 	},
 	{
 	    name: 'matchesFilter',
 	    type: 'boolean',
 	    defaultValue: true,
 	},
-    ]
+    ],
 });
 
 Ext.define('PBS.DataStoreContent', {
@@ -69,7 +68,7 @@ Ext.define('PBS.DataStoreContent', {
 	    view.getStore().setSorters([
 		'backup-group',
 		'text',
-		'backup-time'
+		'backup-time',
 	    ]);
 	    Proxmox.Utils.monStoreErrors(view, this.store);
 	    this.reload(); // initial load
@@ -87,7 +86,7 @@ Ext.define('PBS.DataStoreContent', {
 	    this.store.setProxy({
 		type: 'proxmox',
 		timeout: 300*1000, // 5 minutes, we should make that api call faster
-		url:  url
+		url: url,
 	    });
 
 	    this.store.load();
@@ -123,7 +122,7 @@ Ext.define('PBS.DataStoreContent', {
 		    expanded: false,
 		    backup_type: item.data["backup-type"],
 		    backup_id: item.data["backup-id"],
-		    children: []
+		    children: [],
 		};
 	    }
 
@@ -162,7 +161,7 @@ Ext.define('PBS.DataStoreContent', {
 		    }
 		    return false;
 		},
-		after: () => {},
+		after: Ext.emptyFn,
 	    });
 
 	    for (const item of records) {
@@ -180,7 +179,7 @@ Ext.define('PBS.DataStoreContent', {
 
 		data.children = [];
 		for (const file of data.files) {
-		    file.text = file.filename,
+		    file.text = file.filename;
 		    file['crypt-mode'] = PBS.Utils.cryptmap.indexOf(file['crypt-mode']);
 		    file.leaf = true;
 		    file.matchesFilter = true;
@@ -191,6 +190,7 @@ Ext.define('PBS.DataStoreContent', {
 		children.push(data);
 	    }
 
+	    let nowSeconds = Date.now() / 1000;
 	    let children = [];
 	    for (const [name, group] of Object.entries(groups)) {
 		let last_backup = 0;
@@ -200,7 +200,13 @@ Ext.define('PBS.DataStoreContent', {
 		    'sign-only': 0,
 		    encrypt: 0,
 		};
-		for (const item of group.children) {
+		let verify = {
+		    outdated: 0,
+		    none: 0,
+		    failed: 0,
+		    ok: 0,
+		};
+		for (let item of group.children) {
 		    crypt[PBS.Utils.cryptmap[item['crypt-mode']]]++;
 		    if (item["backup-time"] > last_backup && item.size !== null) {
 			last_backup = item["backup-time"];
@@ -208,9 +214,24 @@ Ext.define('PBS.DataStoreContent', {
 			group.files = item.files;
 			group.size = item.size;
 			group.owner = item.owner;
+			verify.lastFailed = item.verification && item.verification.state !== 'ok';
+		    }
+		    if (!item.verification) {
+			verify.none++;
+		    } else {
+			if (item.verification.state === 'ok') {
+			    verify.ok++;
+			} else {
+			    verify.failed++;
+			}
+			let task = Proxmox.Utils.parse_task_upid(item.verification.upid);
+			item.verification.lastTime = task.starttime;
+			if (nowSeconds - task.starttime > 30 * 24 * 60 * 60) {
+			    verify.outdated++;
+			}
 		    }
-
 		}
+		group.verification = verify;
 		group.count = group.children.length;
 		group.matchesFilter = true;
 		crypt.count = group.count;
@@ -221,7 +242,7 @@ Ext.define('PBS.DataStoreContent', {
 
 	    view.setRootNode({
 		expanded: true,
-		children: children
+		children: children,
 	    });
 
 	    if (selected !== undefined) {
@@ -241,13 +262,13 @@ Ext.define('PBS.DataStoreContent', {
 	    Proxmox.Utils.setErrorMask(view, false);
 	    if (view.getStore().getFilters().length > 0) {
 		let searchBox = me.lookup("searchbox");
-		let searchvalue = searchBox.getValue();;
+		let searchvalue = searchBox.getValue();
 		me.search(searchBox, searchvalue);
 	    }
 	},
 
 	onPrune: function(view, rI, cI, item, e, rec) {
-	    var view = this.getView();
+	    view = this.getView();
 
 	    if (!(rec && rec.data)) return;
 	    let data = rec.data;
@@ -265,7 +286,8 @@ Ext.define('PBS.DataStoreContent', {
 	},
 
 	onVerify: function(view, rI, cI, item, e, rec) {
-	    var view = this.getView();
+	    let me = this;
+	    view = me.getView();
 
 	    if (!view.datastore) return;
 
@@ -297,6 +319,7 @@ Ext.define('PBS.DataStoreContent', {
 		success: function(response, options) {
 		    Ext.create('Proxmox.window.TaskViewer', {
 			upid: response.result.data,
+			taskDone: () => me.reload(),
 		    }).show();
 		},
 	    });
@@ -304,7 +327,7 @@ Ext.define('PBS.DataStoreContent', {
 
 	onForget: function(view, rI, cI, item, e, rec) {
 	    let me = this;
-	    var view = this.getView();
+	    view = this.getView();
 
 	    if (!(rec && rec.data)) return;
 	    let data = rec.data;
@@ -359,7 +382,8 @@ Ext.define('PBS.DataStoreContent', {
 	    let atag = document.createElement('a');
 	    params['file-name'] = file;
 	    atag.download = filename;
-	    let url = new URL(`/api2/json/admin/datastore/${view.datastore}/download-decoded`, window.location.origin);
+	    let url = new URL(`/api2/json/admin/datastore/${view.datastore}/download-decoded`,
+	                      window.location.origin);
 	    for (const [key, value] of Object.entries(params)) {
 		url.searchParams.append(key, value);
 	    }
@@ -422,7 +446,7 @@ Ext.define('PBS.DataStoreContent', {
 		store.beginUpdate();
 		store.getRoot().cascadeBy({
 		    before: function(item) {
-			if(me.filter(item, value)) {
+			if (me.filter(item, value)) {
 			    item.set('matchesFilter', true);
 			    if (item.parentNode && item.parentNode.id !== 'root') {
 				item.parentNode.childmatches = true;
@@ -454,12 +478,22 @@ Ext.define('PBS.DataStoreContent', {
 	},
     },
 
+    viewConfig: {
+	getRowClass: function(record, index) {
+	    let verify = record.get('verification');
+	    if (verify && verify.lastFailed) {
+		return 'proxmox-invalid-row';
+	    }
+	    return null;
+	},
+    },
+
     columns: [
 	{
 	    xtype: 'treecolumn',
 	    header: gettext("Backup Group"),
 	    dataIndex: 'text',
-	    flex: 1
+	    flex: 1,
 	},
 	{
 	    header: gettext('Actions'),
@@ -506,9 +540,9 @@ Ext.define('PBS.DataStoreContent', {
 			    data.filename &&
 			    data.filename.endsWith('pxar.didx') &&
 			    data['crypt-mode'] < 3);
-		    }
+		    },
 		},
-	    ]
+	    ],
 	},
 	{
 	    xtype: 'datecolumn',
@@ -516,7 +550,7 @@ Ext.define('PBS.DataStoreContent', {
 	    sortable: true,
 	    dataIndex: 'backup-time',
 	    format: 'Y-m-d H:i:s',
-	    width: 150
+	    width: 150,
 	},
 	{
 	    header: gettext("Size"),
@@ -538,6 +572,8 @@ Ext.define('PBS.DataStoreContent', {
 	    format: '0',
 	    header: gettext("Count"),
 	    sortable: true,
+	    width: 75,
+	    align: 'right',
 	    dataIndex: 'count',
 	},
 	{
@@ -560,8 +596,80 @@ Ext.define('PBS.DataStoreContent', {
 		if (iconCls) {
 		    iconTxt = `<i class="fa fa-fw fa-${iconCls}"></i> `;
 		}
-		return (iconTxt + PBS.Utils.cryptText[v]) || Proxmox.Utils.unknownText
+		return (iconTxt + PBS.Utils.cryptText[v]) || Proxmox.Utils.unknownText;
-	    }
+	    },
+	},
+	{
+	    header: gettext('Verify State'),
+	    sortable: true,
+	    dataIndex: 'verification',
+	    width: 120,
+	    renderer: (v, meta, record) => {
+		let i = (cls, txt) => `<i class="fa fa-fw fa-${cls}"></i> ${txt}`;
+		if (v === undefined || v === null) {
+		    return record.data.leaf ? '' : i('question-circle-o warning', gettext('None'));
+		}
+		let tip, iconCls, txt;
+		if (record.parentNode.id === 'root') {
+		    if (v.failed === 0) {
+			if (v.none === 0) {
+			    if (v.outdated > 0) {
+				tip = 'All OK, but some snapshots were not verified in last 30 days';
+				iconCls = 'check warning';
+				txt = gettext('All OK (old)');
+			    } else {
+				tip = 'All snapshots verified at least once in last 30 days';
+				iconCls = 'check good';
+				txt = gettext('All OK');
+			    }
+			} else if (v.ok === 0) {
+			    tip = `${v.none} not verified yet`;
+			    iconCls = 'question-circle-o warning';
+			    txt = gettext('None');
+			} else {
+			    tip = `${v.ok} OK, ${v.none} not verified yet`;
+			    iconCls = 'check faded';
+			    txt = `${v.ok} OK`;
+			}
+		    } else {
+			tip = `${v.ok} OK, ${v.failed} failed, ${v.none} not verified yet`;
+			iconCls = 'times critical';
+			txt = v.ok === 0 && v.none === 0
+			    ? gettext('All failed')
+			    : `${v.failed} failed`;
+		    }
+		} else if (!v.state) {
+		    return record.data.leaf ? '' : gettext('None');
+		} else {
+		    let verify_time = Proxmox.Utils.render_timestamp(v.lastTime);
+		    tip = `Last verify task started on ${verify_time}`;
+		    txt = v.state;
+		    iconCls = 'times critical';
+		    if (v.state === 'ok') {
+			iconCls = 'check good';
+			let now = Date.now() / 1000;
+			if (now - v.lastTime > 30 * 24 * 60 * 60) {
+			    tip = `Last verify task over 30 days ago: ${verify_time}`;
+			    iconCls = 'check warning';
+			}
+		    }
+		}
+		return `<span data-qtip="${tip}">
+		    <i class="fa fa-fw fa-${iconCls}"></i> ${txt}
+		</span>`;
+	    },
+	    listeners: {
+		dblclick: function(view, el, row, col, ev, rec) {
+		    let data = rec.data || {};
+		    let verify = data.verification;
+		    if (verify && verify.upid && rec.parentNode.id !== 'root') {
+			let win = Ext.create('Proxmox.window.TaskViewer', {
+			    upid: verify.upid,
+			});
+			win.show();
+		    }
+		},
+	    },
 	},
     ],
 
@@ -579,6 +687,7 @@ Ext.define('PBS.DataStoreContent', {
 	{
 	    xtype: 'textfield',
 	    reference: 'searchbox',
+	    emptyText: gettext('group, date or owner'),
 	    triggers: {
 		clear: {
 		    cls: 'pmx-clear-trigger',
@@ -588,7 +697,7 @@ Ext.define('PBS.DataStoreContent', {
 			this.triggers.clear.setVisible(false);
 			this.setValue('');
 		    },
-		}
+		},
 	    },
 	    listeners: {
 		change: {
@@ -596,6 +705,6 @@ Ext.define('PBS.DataStoreContent', {
 		    buffer: 500,
 		},
 	    },
-	}
+	},
     ],
 });

www/Makefile

@@ -54,6 +54,11 @@ all: js/proxmox-backup-gui.js css/ext6-pbs.css
 js:
 	mkdir js
 
+.PHONY: OnlineHelpInfo.js
+OnlineHelpInfo.js:
+	$(MAKE) -C ../docs onlinehelpinfo
+	mv ../docs/output/scanrefs/OnlineHelpInfo.js .
+
 js/proxmox-backup-gui.js: js OnlineHelpInfo.js ${JSSRC}
 	cat OnlineHelpInfo.js ${JSSRC} >$@.tmp
 	mv $@.tmp $@

www/OnlineHelpInfo.js

@@ -1,6 +1,10 @@
-var proxmoxOnlineHelpInfo = {
+const proxmoxOnlineHelpInfo = {
-    "pbs_documentation_index" : {
+  "pbs_documentation_index": {
-	"link" : "/pbs-docs/index.html",
+    "link": "/docs/index.html",
-	"title" : "Proxmox Backup Server Documentation Index"
+    "title": "Proxmox Backup Server Documentation Index"
-    }
+  },
+  "chapter-zfs": {
+    "link": "/docs/sysadmin.html#chapter-zfs",
+    "title": "ZFS on Linux"
+  }
 };

www/Utils.js

@@ -41,9 +41,9 @@ Ext.define('PBS.Utils', {
 	let files = data.count;
 	if (mixed > 0) {
 	    return PBS.Utils.cryptmap.indexOf('mixed');
-	} else if (files === encrypted) {
+	} else if (files === encrypted && encrypted > 0) {
 	    return PBS.Utils.cryptmap.indexOf('encrypt');
-	} else if (files === signed) {
+	} else if (files === signed && signed > 0) {
 	    return PBS.Utils.cryptmap.indexOf('sign-only');
 	} else if ((signed+encrypted) === 0) {
 	    return PBS.Utils.cryptmap.indexOf('none');

www/form/CalendarEvent.js

@@ -4,15 +4,17 @@ Ext.define('PBS.data.CalendarEventExamples', {
 
     field: ['value', 'text'],
     data: [
-	//FIXME { value: '*/30', text: Ext.String.format(gettext("Every {0} minutes"), 30) },
+	{ value: '*/30', text: Ext.String.format(gettext("Every {0} minutes"), 30) },
 	{ value: 'hourly', text: gettext("Every hour") },
-	//FIXME { value: '*/2:00', text: gettext("Every two hours") },
+	{ value: '*/2:00', text: gettext("Every two hours") },
 	{ value: '2,22:30', text: gettext("Every day") + " 02:30, 22:30" },
 	{ value: 'daily', text: gettext("Every day") + " 00:00" },
 	{ value: 'mon..fri', text: gettext("Monday to Friday") + " 00:00" },
-	//FIXME{ value: 'mon..fri */1:00', text: gettext("Monday to Friday") + ': ' + gettext("hourly") },
+	{ value: 'mon..fri *:00', text: gettext("Monday to Friday") + ', ' + gettext("hourly") },
 	{ value: 'sat 18:15', text: gettext("Every Saturday") + " 18:15" },
-	//FIXME{ value: 'monthly', text: gettext("Every 1st of Month") + " 00:00" }, // not yet possible..
+	{ value: 'monthly', text: gettext("Every first day of the Month") + " 00:00" },
+	{ value: 'sat *-1..7 02:00', text: gettext("Every first Saturday of the month") + " 02:00" },
+	{ value: 'yearly', text: gettext("First day of the year") + " 00:00" },
     ],
 });
 
@@ -26,6 +28,8 @@ Ext.define('PBS.form.CalendarEvent', {
     displayField: 'text',
     queryMode: 'local',
 
+    matchFieldWidth: false,
+
     config: {
 	deleteEmpty: true,
     },

www/index.hbs

@@ -12,7 +12,11 @@
     <link rel="stylesheet" type="text/css" href="/fontawesome/css/font-awesome.css" />
     <link rel="stylesheet" type="text/css" href="/widgettoolkit/css/ext6-pmx.css" />
     <link rel="stylesheet" type="text/css" href="/css/ext6-pbs.css" />
+    {{#if language}}
+    <script type='text/javascript' src='/locale/pbs-lang-{{ language }}.js'></script>
+    {{else}}
     <script type='text/javascript'> function gettext(buf) { return buf; } </script>
+    {{/if}}
     {{#if debug}}
     <script type="text/javascript" src="/extjs/ext-all-debug.js"></script>
     <script type="text/javascript" src="/extjs/charts-debug.js"></script>