bump version to 0.9.0-1

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "proxmox-backup"
-version = "0.8.13"
+version = "0.9.0"
 authors = ["Dietmar Maurer <dietmar@proxmox.com>"]
 edition = "2018"
 license = "AGPL-3"
@@ -18,7 +18,6 @@ apt-pkg-native = "0.3.1" # custom patched version
 base64 = "0.12"
 bitflags = "1.2.1"
 bytes = "0.5"
-chrono = "0.4" # Date and time library for Rust
 crc32fast = "1"
 endian_trait = { version = "0.6", features = ["arrays"] }
 anyhow = "1.0"
@@ -26,7 +25,7 @@ futures = "0.3"
 h2 = { version = "0.2", features = ["stream"] }
 handlebars = "3.0"
 http = "0.2"
-hyper = "0.13"
+hyper = "0.13.6"
 lazy_static = "1.4"
 libc = "0.2"
 log = "0.4"
@@ -39,11 +38,11 @@ pam-sys = "0.5"
 percent-encoding = "2.1"
 pin-utils = "0.1.0"
 pathpatterns = "0.1.2"
-proxmox = { version = "0.3.3", features = [ "sortable-macro", "api-macro", "websocket" ] }
+proxmox = { version = "0.4.2", features = [ "sortable-macro", "api-macro", "websocket" ] }
 #proxmox = { git = "ssh://gitolite3@proxdev.maurer-it.com/rust/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
 #proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro", "websocket" ] }
 proxmox-fuse = "0.1.0"
-pxar = { version = "0.6.0", features = [ "tokio-io", "futures-io" ] }
+pxar = { version = "0.6.1", features = [ "tokio-io", "futures-io" ] }
 #pxar = { path = "../pxar", features = [ "tokio-io", "futures-io" ] }
 regex = "1.2"
 rustyline = "6"
@@ -62,6 +61,7 @@ walkdir = "2"
 xdg = "2.2"
 zstd = { version = "0.4", features = [ "bindgen" ] }
 nom = "5.1"
+crossbeam-channel = "0.4"
 
 [features]
 default = []

Makefile

@@ -150,4 +150,4 @@ upload: ${SERVER_DEB} ${CLIENT_DEB} ${DOC_DEB}
 	# check if working directory is clean
 	git diff --exit-code --stat && git diff --exit-code --stat --staged
 	tar cf - ${SERVER_DEB} ${SERVER_DBG_DEB} ${DOC_DEB} | ssh -X repoman@repo.proxmox.com upload --product pbs --dist buster
-	tar cf - ${CLIENT_DEB} ${CLIENT_DBG_DEB} | ssh -X repoman@repo.proxmox.com upload --product "pbs,pve" --dist buster
+	tar cf - ${CLIENT_DEB} ${CLIENT_DBG_DEB} | ssh -X repoman@repo.proxmox.com upload --product "pbs,pve,pmg" --dist buster

debian/changelog

@@ -1,3 +1,175 @@
+rust-proxmox-backup (0.9.0-1) unstable; urgency=medium
+
+  * use ParallelHandler to verify chunks
+
+  * client: add new paper-key command to CLI tool
+
+  * server: split task list in active and archived
+
+  * tools: add logrotate module and use it for archived tasks, allowing to save
+    more than 100 thousands of tasks efficiently in the archive
+
+  * require square [brackets] for ipv6 addresses and fix ipv6 handling for
+    remotes/sync jobs
+
+  * ui: RemoteEdit: make comment and fingerprint deletable
+
+  * api/disks: create zfs: enable import systemd service unit for newly created
+    ZFS pools
+
+  * client and remotes: add support to specify a custom port number. The server
+    is still always listening on 8007, but you can now use things like reverse
+    proxies or port mapping.
+
+  * ui: RemoteEdit: allow to specify a port in the host field
+
+  * client pull: log progress
+
+  * various fixes and improvements
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 01 Oct 2020 16:19:40 +0200
+
+rust-proxmox-backup (0.8.21-1) unstable; urgency=medium
+
+  * depend on crossbeam-channel
+
+  * speedup sync jobs (allow up to 4 worker threads)
+
+  * improve docs
+
+  * use jobstate mechanism for verify/garbage_collection schedules
+
+  * proxy: fix error handling in prune scheduling
+
+ -- Proxmox Support Team <support@proxmox.com>  Fri, 25 Sep 2020 13:20:19 +0200
+
+rust-proxmox-backup (0.8.20-1) unstable; urgency=medium
+
+  * improve sync speed
+
+  * benchmark: use compressable data to get more realistic result
+
+  * docs: add onlineHelp to some panels
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 24 Sep 2020 13:15:45 +0200
+
+rust-proxmox-backup (0.8.19-1) unstable; urgency=medium
+
+  * src/api2/reader.rs: use std::fs::read instead of tokio::fs::read
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 22 Sep 2020 13:30:27 +0200
+
+rust-proxmox-backup (0.8.18-1) unstable; urgency=medium
+
+  * src/client/pull.rs: allow up to 20 concurrent download streams
+
+  * docs: add version and date to HTML index
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 22 Sep 2020 12:39:26 +0200
+
+rust-proxmox-backup (0.8.17-1) unstable; urgency=medium
+
+  * src/client/pull.rs: open temporary manifest with truncate(true)
+
+  * depend on proxmox 0.4.1
+
+  * fix #3017: check array boundaries before using
+
+  * datastore/prune schedules: use JobState for tracking of schedules
+
+  * improve docs
+
+  * fix #3015: allow user self-service
+
+  * add verification scheduling to proxmox-backup-proxy
+
+  * fix #3014: allow DataStoreAdmins to list DS config
+
+  * depend on pxar 0.6.1
+
+  * fix #2942: implement lacp bond mode and bond_xmit_hash_policy
+
+  * api2/pull: make pull worker abortable
+
+  * fix #2870: renew tickets in HttpClient
+
+  * always allow retrieving (censored) subscription info
+
+  * fix #2957: allow Sys.Audit access to node RRD
+
+  * backup: check all referenced chunks actually exist
+
+  * backup: check verify state of previous backup before allowing reuse
+
+  * avoid chrono dependency
+
+ -- Proxmox Support Team <support@proxmox.com>  Mon, 21 Sep 2020 14:08:32 +0200
+
+rust-proxmox-backup (0.8.16-1) unstable; urgency=medium
+
+  * BackupDir: make constructor fallible
+
+  * handle invalid mtime when formating entries
+
+  * ui/docs: add onlineHelp button for syncjobs
+
+  * docs: add section for calendar events
+
+  * tools/systemd/parse_time: enable */x syntax for calendar events
+
+  * docs: set html img width limitation through css
+
+  * docs: use alabaster theme
+
+  * server: set http2 max frame size
+
+  * doc: Add section "FAQ"
+
+ -- Proxmox Support Team <support@proxmox.com>  Fri, 11 Sep 2020 15:54:57 +0200
+
+rust-proxmox-backup (0.8.15-1) unstable; urgency=medium
+
+  *  verify: skip benchmark directory
+
+  * add benchmark flag to backup creation for proper cleanup when running
+    a benchmark
+
+  * mount: fix mount subcommand
+
+  * ui: only mark backup encrypted if there are any files
+
+  * fix #2983: improve tcp performance
+
+  * improve ui and docs
+
+  * verify: rename corrupted chunks with .bad extension
+
+  * gc: remove .bad files on garbage collect
+
+  * ui: add translation support
+
+  * server/worker_task: fix upid_read_status
+
+  * tools/systemd/time: enable dates for calendarevents
+
+  * server/worker_task: fix 'unknown' status for some big task logs
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 10 Sep 2020 09:25:59 +0200
+
+rust-proxmox-backup (0.8.14-1) unstable; urgency=medium
+
+  * verify speed up: use separate IO thread, use datastore-wide cache (instead
+    of per group)
+
+  * ui: datastore content: improve encrypted column
+
+  * ui: datastore content: show more granular verify state, especially for
+    backup group rows
+
+  * verify: log progress in percent
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 02 Sep 2020 09:36:47 +0200
+
 rust-proxmox-backup (0.8.13-1) unstable; urgency=medium
 
   * improve and add to documentation
@@ -429,4 +601,3 @@ proxmox-backup (0.1-1) unstable; urgency=medium
   * first try
 
  -- Proxmox Support Team <support@proxmox.com>  Fri, 30 Nov 2018 13:03:28 +0100
-

debian/control

@@ -11,8 +11,8 @@ Build-Depends: debhelper (>= 11),
  librust-base64-0.12+default-dev,
  librust-bitflags-1+default-dev (>= 1.2.1-~~),
  librust-bytes-0.5+default-dev,
- librust-chrono-0.4+default-dev,
  librust-crc32fast-1+default-dev,
+ librust-crossbeam-channel-0.4+default-dev,
  librust-endian-trait-0.6+arrays-dev,
  librust-endian-trait-0.6+default-dev,
  librust-futures-0.3+default-dev,
@@ -20,7 +20,7 @@ Build-Depends: debhelper (>= 11),
  librust-h2-0.2+stream-dev,
  librust-handlebars-3+default-dev,
  librust-http-0.2+default-dev,
- librust-hyper-0.13+default-dev,
+ librust-hyper-0.13+default-dev (>= 0.13.6-~~),
  librust-lazy-static-1+default-dev (>= 1.4-~~),
  librust-libc-0.2+default-dev,
  librust-log-0.4+default-dev,
@@ -34,14 +34,14 @@ Build-Depends: debhelper (>= 11),
  librust-pathpatterns-0.1+default-dev (>= 0.1.2-~~),
  librust-percent-encoding-2+default-dev (>= 2.1-~~),
  librust-pin-utils-0.1+default-dev,
- librust-proxmox-0.3+api-macro-dev (>= 0.3.3-~~),
- librust-proxmox-0.3+default-dev (>= 0.3.3-~~),
- librust-proxmox-0.3+sortable-macro-dev (>= 0.3.3-~~),
- librust-proxmox-0.3+websocket-dev (>= 0.3.3-~~),
+ librust-proxmox-0.4+api-macro-dev (>= 0.4.2-~~),
+ librust-proxmox-0.4+default-dev (>= 0.4.2-~~),
+ librust-proxmox-0.4+sortable-macro-dev (>= 0.4.2-~~),
+ librust-proxmox-0.4+websocket-dev (>= 0.4.2-~~),
  librust-proxmox-fuse-0.1+default-dev,
- librust-pxar-0.6+default-dev,
- librust-pxar-0.6+futures-io-dev,
- librust-pxar-0.6+tokio-io-dev,
+ librust-pxar-0.6+default-dev (>= 0.6.1-~~),
+ librust-pxar-0.6+futures-io-dev (>= 0.6.1-~~),
+ librust-pxar-0.6+tokio-io-dev (>= 0.6.1-~~),
  librust-regex-1+default-dev (>= 1.2-~~),
  librust-rustyline-6+default-dev,
  librust-serde-1+default-dev,
@@ -78,6 +78,7 @@ Build-Depends: debhelper (>= 11),
  uuid-dev,
  debhelper (>= 12~),
  bash-completion,
+ pve-eslint,
  python3-docutils,
  python3-pygments,
  rsync,
@@ -103,6 +104,7 @@ Depends: fonts-font-awesome,
          libjs-extjs (>= 6.0.1),
          libzstd1 (>= 1.3.8),
          lvm2,
+         pbs-i18n,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
          proxmox-widget-toolkit (>= 2.2-4),
@@ -117,7 +119,7 @@ Description: Proxmox Backup Server daemon with tools and GUI
 
 Package: proxmox-backup-client
 Architecture: any
-Depends: ${misc:Depends}, ${shlibs:Depends}
+Depends: qrencode ${misc:Depends}, ${shlibs:Depends}
 Description: Proxmox Backup Client tools
  This package contains the Proxmox Backup client, which provides a
  simple command line tool to create and restore backups.

debian/control.in

@@ -4,9 +4,10 @@ Depends: fonts-font-awesome,
          libjs-extjs (>= 6.0.1),
          libzstd1 (>= 1.3.8),
          lvm2,
+         pbs-i18n,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
-         proxmox-widget-toolkit (>= 2.2-4),
+         proxmox-widget-toolkit (>= 2.3-1),
          pve-xtermjs (>= 4.7.0-1),
          smartmontools,
          ${misc:Depends},
@@ -18,7 +19,7 @@ Description: Proxmox Backup Server daemon with tools and GUI
 
 Package: proxmox-backup-client
 Architecture: any
-Depends: ${misc:Depends}, ${shlibs:Depends}
+Depends: qrencode ${misc:Depends}, ${shlibs:Depends}
 Description: Proxmox Backup Client tools
  This package contains the Proxmox Backup client, which provides a
  simple command line tool to create and restore backups.

debian/debcargo.toml

@@ -14,6 +14,7 @@ section = "admin"
 build_depends = [
   "debhelper (>= 12~)",
   "bash-completion",
+  "pve-eslint",
   "python3-docutils",
   "python3-pygments",
   "rsync",

debian/postinst

@@ -15,11 +15,10 @@ case "$1" in
 	fi
 	deb-systemd-invoke $_dh_action proxmox-backup.service proxmox-backup-proxy.service >/dev/null || true
 
-	if test -n "$2"; then
-		if dpkg --compare-versions "$2" 'le' '0.8.10-1'; then
-			echo "Fixing up termproxy user id in task log..."
-			flock -w 30 /var/log/proxmox-backup/tasks/active.lock sed -i 's/:termproxy::root: /:termproxy::root@pam: /' /var/log/proxmox-backup/tasks/active
-		fi
+	# FIXME: Remove in future version once we're sure no broken entries remain in anyone's files
+	if grep -q -e ':termproxy::[^@]\+: ' /var/log/proxmox-backup/tasks/active; then
+	    echo "Fixing up termproxy user id in task log..."
+	    flock -w 30 /var/log/proxmox-backup/tasks/active.lock sed -i 's/:termproxy::\([^@]\+\): /:termproxy::\1@pam: /' /var/log/proxmox-backup/tasks/active
 	fi
     ;;
 

docs/Makefile

@@ -28,7 +28,6 @@ COMPILEDIR := ../target/debug
 SPHINXOPTS    += -t devbuild
 endif
 
-
 # Sphinx internal variables.
 ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) .
 
@@ -68,9 +67,17 @@ proxmox-backup-manager.1: proxmox-backup-manager/man1.rst  proxmox-backup-manage
 proxmox-backup-proxy.1: proxmox-backup-proxy/man1.rst  proxmox-backup-proxy/description.rst
 	rst2man $< >$@
 
+.PHONY: onlinehelpinfo
+onlinehelpinfo:
+	@echo "Generating OnlineHelpInfo.js..."
+	$(SPHINXBUILD) -b proxmox-scanrefs $(ALLSPHINXOPTS) $(BUILDDIR)/scanrefs
+	@echo "Build finished. OnlineHelpInfo.js is in $(BUILDDIR)/scanrefs."
+
 .PHONY: html
-html: ${GENERATED_SYNOPSIS}
+html: ${GENERATED_SYNOPSIS} images/proxmox-logo.svg custom.css conf.py
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	cp images/proxmox-logo.svg $(BUILDDIR)/html/_static/
+	cp custom.css $(BUILDDIR)/html/_static/
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 

docs/_ext/proxmox-scanrefs.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+# debugging stuff
+from pprint import pprint
+
+from typing import cast
+
+import json
+import re
+
+import os
+import io
+from docutils import nodes
+
+from sphinx.builders import Builder
+from sphinx.util import logging
+
+logger = logging.getLogger(__name__)
+
+# refs are added in the following manner before the title of a section (note underscore and newline before title):
+# .. _my-label:
+#
+# Section to ref
+# --------------
+#
+#
+# then referred to like (note missing underscore):
+# "see :ref:`my-label`"
+#
+# the benefit of using this is if a label is explicitly set for a section,
+# we can refer to it with this anchor #my-label in the html,
+# even if the section name changes.
+#
+# see https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref
+
+def scan_extjs_files(wwwdir="../www"): # a bit rough i know, but we can optimize later
+    js_files = []
+    used_anchors = []
+    logger.info("scanning extjs files for onlineHelp definitions")
+    for root, dirs, files in os.walk("{}".format(wwwdir)):
+        #print(root, dirs, files)
+        for filename in files:
+            if filename.endswith('.js'):
+                js_files.append(os.path.join(root, filename))
+    for js_file in js_files:
+        fd = open(js_file).read()
+        match = re.search("onlineHelp:\s*[\'\"](.*?)[\'\"]", fd) # match object is tuple
+        if match:
+            anchor = match.groups()[0]
+            anchor = re.sub('_', '-', anchor) # normalize labels
+            logger.info("found onlineHelp: {} in {}".format(anchor, js_file))
+            used_anchors.append(anchor)
+    return used_anchors
+
+
+def setup(app):
+    logger.info('Mapping reference labels...')
+    app.add_builder(ReflabelMapper)
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }
+
+class ReflabelMapper(Builder):
+    name = 'proxmox-scanrefs'
+
+    def init(self):
+        self.docnames = []
+        self.env.online_help = {}
+        self.env.online_help['pbs_documentation_index'] = {
+            'link': '/docs/index.html',
+            'title': 'Proxmox Backup Server Documentation Index',
+        }
+        self.env.used_anchors = scan_extjs_files()
+
+        if not os.path.isdir(self.outdir):
+            os.mkdir(self.outdir)
+
+        self.output_filename = os.path.join(self.outdir, 'OnlineHelpInfo.js')
+        self.output = io.open(self.output_filename, 'w', encoding='UTF-8')
+
+    def write_doc(self, docname, doctree):
+            for node in doctree.traverse(nodes.section):
+                #pprint(vars(node))
+
+                if hasattr(node, 'expect_referenced_by_id') and len(node['ids']) > 1: # explicit labels
+                    filename = self.env.doc2path(docname)
+                    filename_html = re.sub('.rst', '.html', filename)
+                    labelid = node['ids'][1] # [0] is predefined by sphinx, we need [1] for explicit ones
+                    title = cast(nodes.title, node[0])
+                    logger.info('traversing section {}'.format(title.astext()))
+                    ref_name = getattr(title, 'rawsource', title.astext())
+
+                    self.env.online_help[labelid] = {'link': '', 'title': ''}
+                    self.env.online_help[labelid]['link'] = "/docs/" + os.path.basename(filename_html) + "#{}".format(labelid)
+                    self.env.online_help[labelid]['title'] = ref_name
+
+            return
+
+
+    def get_outdated_docs(self):
+        return 'all documents'
+
+    def prepare_writing(self, docnames):
+        return
+
+    def get_target_uri(self, docname, typ=None):
+        return ''
+
+    def validate_anchors(self):
+        #pprint(self.env.online_help)
+        to_remove = []
+        for anchor in self.env.used_anchors:
+            if anchor not in self.env.online_help:
+                logger.info("[-] anchor {} is missing from onlinehelp!".format(anchor))
+        for anchor in self.env.online_help:
+            if anchor not in self.env.used_anchors and anchor != 'pbs_documentation_index':
+                logger.info("[*] anchor {} not used! deleting...".format(anchor))
+                to_remove.append(anchor)
+        for anchor in to_remove:
+            self.env.online_help.pop(anchor, None)
+        return
+
+    def finish(self):
+        # generate OnlineHelpInfo.js output
+        self.validate_anchors()
+
+        self.output.write("const proxmoxOnlineHelpInfo = ")
+        self.output.write(json.dumps(self.env.online_help, indent=2))
+        self.output.write(";\n")
+        self.output.close()
+        return

docs/_templates/index-sidebar.html

@@ -0,0 +1,11 @@
+<h3>Navigation</h3>
+{{ toctree(includehidden=theme_sidebar_includehidden, collapse=True, titles_only=True) }}
+{% if theme_extra_nav_links %}
+<hr />
+<h3>Links</h3>
+<ul>
+    {% for text, uri in theme_extra_nav_links.items() %}
+    <li class="toctree-l1"><a href="{{ uri }}">{{ text }}</a></li>
+    {% endfor %}
+</ul>
+{% endif %}

docs/_templates/sidebar-header.html

@@ -0,0 +1,7 @@
+ <p class="logo">
+    <a href="index.html">
+        <img class="logo" src="_static/proxmox-logo.svg" alt="Logo">
+    </a>
+</p>
+<h1 class="logo logo-name"><a href="index.html">Proxmox Backup</a></h1>
+<hr style="width:100%;">

docs/administration-guide.rst

@@ -24,6 +24,13 @@ good deduplication rates for file archives.
 The Proxmox Backup Server supports both strategies.
 
 
+Image Archives: ``<name>.img``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This is used for virtual machine images and other large binary
+data. Content is split into fixed-sized chunks.
+
+
 File Archives: ``<name>.pxar``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -34,13 +41,6 @@ the :ref:`pxar-format`, split into variable-sized chunks. The format
 is optimized to achieve good deduplication rates.
 
 
-Image Archives: ``<name>.img``
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-This is used for virtual machine images and other large binary
-data. Content is split into fixed-sized chunks.
-
-
 Binary Data (BLOBs)
 ^^^^^^^^^^^^^^^^^^^
 
@@ -127,17 +127,18 @@ Backup Server Management
 The command line tool to configure and manage the backup server is called
 :command:`proxmox-backup-manager`.
 
-
+.. _datastore_intro:
 
 :term:`DataStore`
 ~~~~~~~~~~~~~~~~~
 
-A datastore is a place where backups are stored. The current implementation
-uses a directory inside a standard unix file system (``ext4``, ``xfs``
-or ``zfs``) to store the backup data.
+A datastore refers to a location at which backups are stored. The current
+implementation uses a directory inside a standard unix file system (``ext4``,
+``xfs`` or ``zfs``) to store the backup data.
 
-Datastores are identified by a simple *ID*. You can configure it
-when setting up the backup server.
+Datastores are identified by a simple *ID*. You can configure this
+when setting up the datastore. The configuration information for datastores
+is stored in the file ``/etc/proxmox-backup/datastore.cfg``.
 
 .. note:: The `File Layout`_ requires the file system to support at least *65538*
    subdirectories per directory. That number comes from the 2\ :sup:`16`
@@ -148,11 +149,17 @@ when setting up the backup server.
 
 Disk Management
 ~~~~~~~~~~~~~~~
+
+.. image:: images/screenshots/pbs-gui-disks.png
+  :align: right
+  :alt: List of disks
+
 Proxmox Backup Server comes with a set of disk utilities, which are
 accessed using the ``disk`` subcommand. This subcommand allows you to initialize
 disks, create various filesystems, and get information about the disks.
 
-To view the disks connected to the system, use the ``list`` subcommand of
+To view the disks connected to the system, navigate to **Administration ->
+Disks** in the web interface or use the ``list`` subcommand of
 ``disk``:
 
 .. code-block:: console
@@ -174,66 +181,98 @@ To initialize a disk with a new GPT, use the ``initialize`` subcommand:
 
   # proxmox-backup-manager disk initialize sdX
 
-You can create an ``ext4`` or ``xfs`` filesystem on a disk, using ``fs
-create``. The following command creates an ``ext4`` filesystem and passes the
-``--add-datastore`` parameter, in order to automatically create a datastore on
-the disk (in this case ``sdd``). This will create a datastore at the location
-``/mnt/datastore/store1``:
+.. image:: images/screenshots/pbs-gui-disks-dir-create.png
+  :align: right
+  :alt: Create a directory
+
+You can create an ``ext4`` or ``xfs`` filesystem on a disk using ``fs
+create``, or by navigating to **Administration -> Disks -> Directory** in the
+web interface and creating one from there. The following command creates an
+``ext4`` filesystem and passes the ``--add-datastore`` parameter, in order to
+automatically create a datastore on the disk (in this case ``sdd``). This will
+create a datastore at the location ``/mnt/datastore/store1``:
 
 .. code-block:: console
 
   # proxmox-backup-manager disk fs create store1 --disk sdd --filesystem ext4 --add-datastore true
-  create datastore 'store1' on disk sdd
-  Percentage done: 1
-  ...
-  Percentage done: 99
-  TASK OK
 
-You can also create a ``zpool`` with various raid levels. The command below
-creates a mirrored ``zpool`` using two disks (``sdb`` & ``sdc``) and mounts it
-on the root directory (default):
+.. image:: images/screenshots/pbs-gui-disks-zfs-create.png
+  :align: right
+  :alt: Create ZFS
+
+You can also create a ``zpool`` with various raid levels from **Administration
+-> Disks -> Zpool** in the web interface, or by using ``zpool create``. The command
+below creates a mirrored ``zpool`` using two disks (``sdb`` & ``sdc``) and
+mounts it on the root directory (default):
 
 .. code-block:: console
 
   # proxmox-backup-manager disk zpool create zpool1 --devices sdb,sdc --raidlevel mirror
-  create Mirror zpool 'zpool1' on devices 'sdb,sdc'
-  # "zpool" "create" "-o" "ashift=12" "zpool1" "mirror" "sdb" "sdc"
 
-  TASK OK
-
-.. note::
-  You can also pass the ``--add-datastore`` parameter here, to automatically
+.. note:: You can also pass the ``--add-datastore`` parameter here, to automatically
   create a datastore from the disk.
 
 You can use ``disk fs list`` and ``disk zpool list`` to keep track of your
 filesystems and zpools respectively.
 
-If a disk supports S.M.A.R.T. capability, and you have this enabled, you can
-display S.M.A.R.T. attributes using the command:
+Proxmox Backup Server uses the package smartmontools. This is a set of tools
+used to monitor and control the S.M.A.R.T. system for local hard disks. If a
+disk supports S.M.A.R.T. capability, and you have this enabled, you can
+display S.M.A.R.T. attributes from the web interface or by using the command:
 
 .. code-block:: console
 
   # proxmox-backup-manager disk smart-attributes sdX
 
+.. note:: This functionality may also be accessed directly through the use of
+  the ``smartctl`` command, which comes as part of the smartmontools package
+  (see ``man smartctl`` for more details).
+
+
 Datastore Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~
 
+.. image:: images/screenshots/pbs-gui-datastore.png
+  :align: right
+  :alt: Datastore Overview
+
 You can configure multiple datastores. Minimum one datastore needs to be
-configured. The datastore is identified by a simple `name` and points to a
+configured. The datastore is identified by a simple *name* and points to a
 directory on the filesystem. Each datastore also has associated retention
 settings of how many backup snapshots for each interval of ``hourly``,
 ``daily``, ``weekly``, ``monthly``, ``yearly`` as well as a time-independent
 number of backups to keep in that store. :ref:`Pruning <pruning>` and
 :ref:`garbage collection <garbage-collection>` can also be configured to run
-periodically based on a configured :term:`schedule` per datastore.
+periodically based on a configured schedule (see :ref:`calendar-events`) per datastore.
 
-The following command creates a new datastore called ``store1`` on :file:`/backup/disk1/store1`
+Creating a Datastore
+^^^^^^^^^^^^^^^^^^^^
+.. image:: images/screenshots/pbs-gui-datastore-create-general.png
+  :align: right
+  :alt: Create a datastore
+
+You can create a new datastore from the web GUI, by navigating to **Datastore** in
+the menu tree and clicking **Create**. Here:
+
+* *Name* refers to the name of the datastore
+* *Backing Path* is the path to the directory upon which you want to create the
+  datastore
+* *GC Schedule* refers to the time and intervals at which garbage collection
+  runs
+* *Prune Schedule* refers to the frequency at which pruning takes place
+* *Prune Options* set the amount of backups which you would like to keep (see :ref:`Pruning <pruning>`).
+
+Alternatively you can create a new datastore from the command line. The
+following command creates a new datastore called ``store1`` on :file:`/backup/disk1/store1`
 
 .. code-block:: console
 
   # proxmox-backup-manager datastore create store1 /backup/disk1/store1
 
-To list existing datastores run:
+Managing Datastores
+^^^^^^^^^^^^^^^^^^^
+
+To list existing datastores from the command line run:
 
 .. code-block:: console
 
@@ -244,13 +283,15 @@ To list existing datastores run:
   │ store1 │ /backup/disk1/store1 │ This is my default storage. │
   └────────┴──────────────────────┴─────────────────────────────┘
 
-You can change settings of a datastore, for example to set a prune and garbage
-collection schedule or retention settings using ``update`` subcommand and view
-a datastore with the ``show`` subcommand:
+You can change the garbage collection and prune settings of a datastore, by
+editing the datastore from the GUI or by using the ``update`` subcommand. For
+example, the below command changes the garbage collection schedule using the
+``update`` subcommand and prints the properties of the datastore with the
+``show`` subcommand:
 
 .. code-block:: console
 
-  # proxmox-backup-manager datastore update store1 --keep-last 7 --prune-schedule daily --gc-schedule 'Tue 04:27'
+  # proxmox-backup-manager datastore update store1 --gc-schedule 'Tue 04:27'
   # proxmox-backup-manager datastore show store1
   ┌────────────────┬─────────────────────────────┐
   │ Name           │ Value                       │
@@ -323,11 +364,15 @@ directories will store the chunked data after a backup operation has been execut
  276489 drwxr-xr-x 3 backup backup 4.0K Jul  8 12:35 ..
  276490 drwxr-x--- 1 backup backup 1.1M Jul  8 12:35 .
 
-
+.. _user_mgmt:
 
 User Management
 ~~~~~~~~~~~~~~~
 
+.. image:: images/screenshots/pbs-gui-user-management.png
+  :align: right
+  :alt: User management
+
 Proxmox Backup Server supports several authentication realms, and you need to
 choose the realm when you add a new user. Possible realms are:
 
@@ -339,7 +384,8 @@ choose the realm when you add a new user. Possible realms are:
       ``/etc/proxmox-backup/shadow.json``.
 
 After installation, there is a single user ``root@pam``, which
-corresponds to the Unix superuser. You can use the
+corresponds to the Unix superuser. User configuration information is stored in the file
+``/etc/proxmox-backup/user.cfg``. You can use the
 ``proxmox-backup-manager`` command line tool to list or manipulate
 users:
 
@@ -352,19 +398,21 @@ users:
   │ root@pam    │      1 │        │           │          │                │ Superuser          │
   └─────────────┴────────┴────────┴───────────┴──────────┴────────────────┴────────────────────┘
 
+.. image:: images/screenshots/pbs-gui-user-management-add-user.png
+  :align: right
+  :alt: Add a new user
+
 The superuser has full administration rights on everything, so you
-normally want to add other users with less privileges:
+normally want to add other users with less privileges. You can create a new
+user with the ``user create`` subcommand or through the web interface, under
+**Configuration -> User Management**. The ``create`` subcommand lets you specify
+many options like ``--email`` or ``--password``. You can update or change any
+user properties using the ``update`` subcommand later (**Edit** in the GUI):
+
 
 .. code-block:: console
 
   # proxmox-backup-manager user create john@pbs --email john@example.com
-
-The create command lets you specify many options like ``--email`` or
-``--password``. You can update or change any of them using the
-update command later:
-
-.. code-block:: console
-
   # proxmox-backup-manager user update john@pbs --firstname John --lastname Smith
   # proxmox-backup-manager user update john@pbs --comment "An example user."
 
@@ -400,6 +448,8 @@ Or completely remove the user with:
   # proxmox-backup-manager user remove john@pbs
 
 
+.. _user_acl:
+
 Access Control
 ~~~~~~~~~~~~~~
 
@@ -442,9 +492,32 @@ following roles exist:
 **RemoteSyncOperator**
   Is allowed to read data from a remote.
 
-You can use the ``acl`` subcommand to manage and monitor user permissions. For
-example, the command below will add the user ``john@pbs`` as a
-**DatastoreAdmin** for the data store ``store1``, located at ``/backup/disk1/store1``:
+.. image:: images/screenshots/pbs-gui-permissions-add.png
+  :align: right
+  :alt: Add permissions for user
+
+Access permission information is stored in ``/etc/proxmox-backup/acl.cfg``. The
+file contains 5 fields, separated using a colon (':') as a delimiter. A typical
+entry takes the form:
+
+``acl:1:/datastore:john@pbs:DatastoreBackup``
+
+The data represented in each field is as follows:
+
+#. ``acl`` identifier
+#. A ``1`` or ``0``, representing whether propagation is enabled or disabled,
+   respectively
+#. The object on which the permission is set. This can be a specific object
+   (single datastore, remote, etc.) or a top level object, which with
+   propagation enabled, represents all children of the object also.
+#. The user for which the permission is set
+#. The role being set
+
+You can manage datastore permissions from **Configuration -> Permissions** in the
+web interface. Likewise, you can use the ``acl`` subcommand to manage and
+monitor user permissions from the command line. For example, the command below
+will add the user ``john@pbs`` as a **DatastoreAdmin** for the datastore
+``store1``, located at ``/backup/disk1/store1``:
 
 .. code-block:: console
 
@@ -461,100 +534,133 @@ You can monitor the roles of each user using the following command:
    │ john@pbs │ /datastore/disk1 │         1 │ DatastoreAdmin │
    └──────────┴──────────────────┴───────────┴────────────────┘
 
-A single user can be assigned multiple permission sets for different data stores.
+A single user can be assigned multiple permission sets for different datastores.
 
 .. Note::
-  Naming convention is important here. For data stores on the host,
+  Naming convention is important here. For datastores on the host,
   you must use the convention ``/datastore/{storename}``. For example, to set
-  permissions for a data store mounted at ``/mnt/backup/disk4/store2``, you would use
+  permissions for a datastore mounted at ``/mnt/backup/disk4/store2``, you would use
   ``/datastore/store2`` for the path. For remote stores, use the convention
   ``/remote/{remote}/{storename}``, where ``{remote}`` signifies the name of the
-  remote (see `Remote` below) and ``{storename}`` is the name of the data store on
+  remote (see `Remote` below) and ``{storename}`` is the name of the datastore on
   the remote.
 
 Network Management
 ~~~~~~~~~~~~~~~~~~
-Proxmox Backup Server provides an interface for network configuration, through the
-``network`` subcommand. This allows you to carry out some basic network
-management tasks such as adding, configuring and removing network interfaces.
+
+Proxmox Backup Server provides both a web interface and a command line tool for
+network configuration. You can find the configuration options in the web
+interface under the **Network Interfaces** section of the **Configuration** menu
+tree item. The command line tool is accessed via the ``network`` subcommand.
+These interfaces allow you to carry out some basic network management tasks,
+such as adding, configuring, and removing network interfaces.
+
+.. note:: Any changes made to the network configuration are not
+  applied, until you click on **Apply Configuration** or enter the ``network
+  reload`` command. This allows you to make many changes at once. It also allows
+  you to ensure that your changes are correct before applying them, as making a
+  mistake here can render the server inaccessible over the network.
 
 To get a list of available interfaces, use the following command:
 
 .. code-block:: console
 
   # proxmox-backup-manager network list
-  ┌───────┬────────┬───────────┬────────┬─────────┬───────────────────┬──────────────┬──────────────┐
-  │ name  │ type   │ autostart │ method │ method6 │ address           │ gateway      │ ports/slaves │
-  ╞═══════╪════════╪═══════════╪════════╪═════════╪═══════════════════╪══════════════╪══════════════╡
-  │ bond0 │ bond   │         1 │ manual │         │                   │              │ ens18 ens19  │
-  ├───────┼────────┼───────────┼────────┼─────────┼───────────────────┼──────────────┼──────────────┤
-  │ ens18 │ eth    │         1 │ manual │         │                   │              │              │
-  ├───────┼────────┼───────────┼────────┼─────────┼───────────────────┼──────────────┼──────────────┤
-  │ ens19 │ eth    │         1 │ manual │         │                   │              │              │
-  ├───────┼────────┼───────────┼────────┼─────────┼───────────────────┼──────────────┼──────────────┤
-  │ vmbr0 │ bridge │         1 │ static │         │ x.x.x.x/x         │ x.x.x.x      │ bond0        │
-  └───────┴────────┴───────────┴────────┴─────────┴───────────────────┴──────────────┴──────────────┘
+  ┌───────┬────────┬───────────┬────────┬─────────────┬──────────────┬──────────────┐
+  │ name  │ type   │ autostart │ method │ address     │ gateway      │ ports/slaves │
+  ╞═══════╪════════╪═══════════╪════════╪═════════════╪══════════════╪══════════════╡
+  │ bond0 │ bond   │         1 │ static │ x.x.x.x/x   │ x.x.x.x      │ ens18 ens19  │
+  ├───────┼────────┼───────────┼────────┼─────────────┼──────────────┼──────────────┤
+  │ ens18 │ eth    │         1 │ manual │             │              │              │
+  ├───────┼────────┼───────────┼────────┼─────────────┼──────────────┼──────────────┤
+  │ ens19 │ eth    │         1 │ manual │             │              │              │
+  └───────┴────────┴───────────┴────────┴─────────────┴──────────────┴──────────────┘
+
+.. image:: images/screenshots/pbs-gui-network-create-bond.png
+  :align: right
+  :alt: Add a network interface
 
 To add a new network interface, use the ``create`` subcommand with the relevant
-parameters. The following command shows a template for creating a new bridge:
+parameters. For example, you may want to set up a bond, for the purpose of
+network redundancy. The following command shows a template for creating the bond shown
+in the list above:
 
 .. code-block:: console
 
-  # proxmox-backup-manager network create vmbr1 --autostart true --cidr x.x.x.x/x --gateway x.x.x.x --bridge_ports iface_name --type bridge
+  # proxmox-backup-manager network create bond0 --type bond --bond_mode active-backup --slaves ens18,ens19 --autostart true --cidr x.x.x.x/x --gateway x.x.x.x
 
 You can make changes to the configuration of a network interface with the
 ``update`` subcommand:
 
 .. code-block:: console
 
-  # proxmox-backup-manager network update vmbr1 --cidr y.y.y.y/y
+  # proxmox-backup-manager network update bond0 --cidr y.y.y.y/y
 
 You can also remove a network interface:
 
 .. code-block:: console
 
-   # proxmox-backup-manager network remove vmbr1
+   # proxmox-backup-manager network remove bond0
 
-To view the changes made to the network configuration file, before committing
-them, use the command:
+The pending changes for the network configuration file will appear at the bottom of the
+web interface. You can also view these changes, by using the command:
 
 .. code-block:: console
 
   # proxmox-backup-manager network changes
 
-If you would like to cancel all changes at this point, you can do this using:
+If you would like to cancel all changes at this point, you can either click on
+the **Revert** button or use the following command:
 
 .. code-block:: console
 
   # proxmox-backup-manager network revert
 
 If you are happy with the changes and would like to write them into the
-configuration file, the command is:
+configuration file, select **Apply Configuration**. The corresponding command
+is:
 
 .. code-block:: console
 
   # proxmox-backup-manager network reload
 
-You can also configure DNS settings using the ``dns`` subcommand of
+.. note:: This command and corresponding GUI button rely on the ``ifreload``
+  command, from the package ``ifupdown2``. This package is included within the
+  Proxmox Backup Server installation, however, you may have to install it yourself,
+  if you have installed Proxmox Backup Server on top of Debian or Proxmox VE.
+
+You can also configure DNS settings, from the **DNS** section
+of **Configuration** or by using the ``dns`` subcommand of
 ``proxmox-backup-manager``.
 
+.. _backup_remote:
+
 :term:`Remote`
 ~~~~~~~~~~~~~~
 
 A remote refers to a separate Proxmox Backup Server installation and a user on that
 installation, from which you can `sync` datastores to a local datastore with a
-`Sync Job`.
+`Sync Job`. You can configure remotes in the web interface, under **Configuration
+-> Remotes**. Alternatively, you can use the ``remote`` subcommand. The
+configuration information for remotes is stored in the file
+``/etc/proxmox-backup/remote.cfg``.
+
+.. image:: images/screenshots/pbs-gui-remote-add.png
+  :align: right
+  :alt: Add a remote
 
 To add a remote, you need its hostname or ip, a userid and password on the
 remote, and its certificate fingerprint. To get the fingerprint, use the
-``proxmox-backup-manager cert info`` command on the remote.
+``proxmox-backup-manager cert info`` command on the remote, or navigate to
+**Dashboard** in the remote's web interface and select **Show Fingerprint**.
 
 .. code-block:: console
 
   # proxmox-backup-manager cert info |grep Fingerprint
   Fingerprint (sha256): 64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe
 
-Using the information specified above, add the remote with:
+Using the information specified above, you can add a remote from the **Remotes**
+configuration panel, or by using the command:
 
 .. code-block:: console
 
@@ -574,14 +680,23 @@ Use the ``list``, ``show``, ``update``, ``remove`` subcommands of
   └──────┴──────────────┴──────────┴───────────────────────────────────────────┴─────────┘
   # proxmox-backup-manager remote remove pbs2
 
+.. _syncjobs:
 
 Sync Jobs
 ~~~~~~~~~
 
-Sync jobs are configured to pull the contents of a datastore on a `Remote` to a
-local datastore. You can either start the sync job manually on the GUI or
-provide it with a :term:`schedule` to run regularly. The
-``proxmox-backup-manager sync-job`` command is used to manage sync jobs:
+.. image:: images/screenshots/pbs-gui-syncjob-add.png
+  :align: right
+  :alt: Add a Sync Job
+
+Sync jobs are configured to pull the contents of a datastore on a **Remote** to
+a local datastore. You can manage sync jobs under **Configuration -> Sync Jobs**
+in the web interface, or using the ``proxmox-backup-manager sync-job`` command.
+The configuration information for sync jobs is stored at
+``/etc/proxmox-backup/sync.cfg``. To create a new sync job, click the add button
+in the GUI, or use the ``create`` subcommand. After creating a sync job, you can
+either start it manually on the GUI or provide it with a schedule (see
+:ref:`calendar-events`) to run regularly.
 
 .. code-block:: console
 
@@ -595,12 +710,13 @@ provide it with a :term:`schedule` to run regularly. The
   └────────────┴───────┴────────┴──────────────┴───────────┴─────────┘
   # proxmox-backup-manager sync-job remove pbs2-local
 
+
 Garbage Collection
 ~~~~~~~~~~~~~~~~~~
 You can monitor and run :ref:`garbage collection <garbage-collection>` on the
 Proxmox Backup Server using the ``garbage-collection`` subcommand of
 ``proxmox-backup-manager``. You can use the ``start`` subcommand to manually start garbage
-collection on an entire data store and the ``status`` subcommand to see
+collection on an entire datastore and the ``status`` subcommand to see
 attributes relating to the :ref:`garbage collection <garbage-collection>`.
 
 
@@ -616,15 +732,33 @@ Repository Locations
 The client uses the following notation to specify a datastore repository
 on the backup server.
 
-  [[username@]server:]datastore
+  [[username@]server[:port]:]datastore
 
-The default value for ``username`` ist ``root``.  If no server is specified,
+The default value for ``username`` ist ``root@pam``.  If no server is specified,
 the default is the local host (``localhost``).
 
+You can specify a port if your backup server is only reachable on a different
+port (e.g. with NAT and port forwarding).
+
+Note that if the server is an IPv6 address, you have to write it with
+square brackets (e.g. [fe80::01]).
+
 You can pass the repository with the ``--repository`` command
 line option, or by setting the ``PBS_REPOSITORY`` environment
 variable.
 
+Here some examples of valid repositories and the real values
+
+================================ ============ ================== ===========
+Example                          User         Host:Port          Datastore
+================================ ============ ================== ===========
+mydatastore                      ``root@pam`` localhost:8007     mydatastore
+myhostname:mydatastore           ``root@pam`` myhostname:8007    mydatastore
+user@pbs@myhostname:mydatastore  ``user@pbs`` myhostname:8007    mydatastore
+192.168.55.55:1234:mydatastore   ``root@pam`` 192.168.55.55:1234 mydatastore
+[ff80::51]:mydatastore           ``root@pam`` [ff80::51]:8007    mydatastore
+[ff80::51]:1234:mydatastore      ``root@pam`` [ff80::51]:1234    mydatastore
+================================ ============ ================== ===========
 
 Environment Variables
 ~~~~~~~~~~~~~~~~~~~~~
@@ -1210,7 +1344,7 @@ Garbage Collection
 ~~~~~~~~~~~~~~~~~~
 
 The ``prune`` command removes only the backup index files, not the data
-from the data store. This task is left to the garbage collection
+from the datastore. This task is left to the garbage collection
 command. It is recommended to carry out garbage collection on a regular basis.
 
 The garbage collection works in two phases. In the first phase, all
@@ -1307,13 +1441,17 @@ as ``user1@pbs``.
   # pvesm add pbs store2 --server localhost --datastore store2
   # pvesm set store2 --username user1@pbs --password <secret>
 
+.. note:: If you would rather not pass your password as plain text, you can pass
+  the ``--password`` parameter, without any arguments. This will cause the
+  program to prompt you for a password upon entering the command.
+
 If your backup server uses a self signed certificate, you need to add
 the certificate fingerprint to the configuration. You can get the
 fingerprint by running the following command on the backup server:
 
 .. code-block:: console
 
-  # proxmox-backup-manager cert  info |grep Fingerprint
+  # proxmox-backup-manager cert info | grep Fingerprint
   Fingerprint (sha256): 64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe
 
 Please add that fingerprint to your configuration to establish a trust
@@ -1331,6 +1469,10 @@ After that you should be able to see storage status with:
   Name             Type     Status           Total            Used       Available        %
   store2            pbs     active      3905109820      1336687816      2568422004   34.23%
 
+Having added the PBS datastore to `Proxmox VE`_, you can backup VMs and
+containers in the same way you would for any other storage device within the
+environment (see `PVE Admin Guide: Backup and Restore
+<https://pve.proxmox.com/pve-docs/pve-admin-guide.html#chapter_vzdump>`_.
 
 
 .. include:: command-line-tools.rst

docs/calendarevents.rst

@@ -0,0 +1,100 @@
+
+.. _calendar-events:
+
+Calendar Events
+===============
+
+Introduction and Format
+-----------------------
+
+Certain tasks, for example pruning and garbage collection, need to be
+performed on a regular basis. Proxmox Backup Server uses a format inspired
+by the systemd Time and Date Specification (see `systemd.time manpage`_)
+called `calendar events` for its schedules.
+
+`Calendar events` are expressions to specify one or more points in time.
+They are mostly compatible with systemds calendar events.
+
+The general format is as follows:
+
+.. code-block:: console
+  :caption: Calendar event
+
+  [WEEKDAY] [[YEARS-]MONTHS-DAYS] [HOURS:MINUTES[:SECONDS]]
+
+Note that there either has to be at least a weekday, date or time part.
+If the weekday or date part is omitted, all (week)days are included.
+If the time part is omitted, the time 00:00:00 is implied.
+(e.g. '2020-01-01' refers to '2020-01-01 00:00:00')
+
+Weekdays are specified with the abbreviated english version:
+`mon, tue, wed, thu, fri, sat, sun`.
+
+Each field can contain multiple values in the following formats:
+
+* comma-separated: e.g.,  01,02,03
+* as a range: e.g., 01..10
+* as a repetition: e.g, 05/10 (means starting at 5 every 10)
+* and a combination of the above: e.g., 01,05..10,12/02
+* or a `*` for every possible value: e.g., \*:00
+
+There are some special values that have specific meaning:
+
+=================================  ==============================
+Value                              Syntax
+=================================  ==============================
+`minutely`                         `*-*-* *:*:00`
+`hourly`                           `*-*-* *:00:00`
+`daily`                            `*-*-* 00:00:00`
+`weekly`                           `mon *-*-* 00:00:00`
+`monthly`                          `*-*-01 00:00:00`
+`yearly` or `annualy`              `*-01-01 00:00:00`
+`quarterly`                        `*-01,04,07,10-01 00:00:00`
+`semiannually` or `semi-annually`  `*-01,07-01 00:00:00`
+=================================  ==============================
+
+
+Here is a table with some useful examples:
+
+======================== =============================  ===================================
+Example                  Alternative                    Explanation
+======================== =============================  ===================================
+`mon,tue,wed,thu,fri`    `mon..fri`                     Every working day at 00:00
+`sat,sun`                `sat..sun`                     Only on weekends at 00:00
+`mon,wed,fri`            --                             Monday, Wednesday, Friday at 00:00
+`12:05`                  --                             Every day at 12:05 PM
+`*:00/5`                 `0/1:0/5`                      Every five minutes
+`mon..wed *:30/10`       `mon,tue,wed *:30/10`          Monday, Tuesday, Wednesday 30, 40 and 50 minutes after every full hour
+`mon..fri 8..17,22:0/15` --                             Every working day every 15 minutes between 8 AM and 6 PM and between 10 PM and 11 PM
+`fri 12..13:5/20`        `fri 12,13:5/20`               Friday at 12:05, 12:25, 12:45, 13:05, 13:25 and 13:45
+`12,14,16,18,20,22:5`    `12/2:5`                       Every day starting at 12:05 until 22:05, every 2 hours
+`*:*`                    `0/1:0/1`                      Every minute (minimum interval)
+`*-05`                   --                             On the 5th day of every Month
+`Sat *-1..7 15:00`       --                             First Saturday each Month at 15:00
+`2015-10-21`             --                             21st October 2015 at 00:00
+======================== =============================  ===================================
+
+
+Differences to systemd
+----------------------
+
+Not all features of systemd calendar events are implemented:
+
+* no unix timestamps (e.g. `@12345`): instead use date and time to specify
+  a specific point in time
+* no timezone: all schedules use the set timezone on the server
+* no sub-second resolution
+* no reverse day syntax (e.g. 2020-03~01)
+* no repetition of ranges (e.g. 1..10/2)
+
+Notes on scheduling
+-------------------
+
+In `Proxmox Backup`_ scheduling for most tasks is done in the
+`proxmox-backup-proxy`. This daemon checks all job schedules
+if they are due every minute. This means that even if
+`calendar events` can contain seconds, it will only be checked
+once a minute.
+
+Also, all schedules will be checked against the timezone set
+in the `Proxmox Backup`_ server.

docs/conf.py

@@ -18,9 +18,12 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-# import sys
+import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
+# custom extensions
+sys.path.append(os.path.abspath("./_ext"))
+
 # -- Implement custom formatter for code-blocks ---------------------------
 #
 # * use smaller font
@@ -46,7 +49,7 @@ PygmentsBridge.latex_formatter = CustomLatexFormatter
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 
-extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo"]
+extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo", "proxmox-scanrefs"]
 
 todo_link_only = True
 
@@ -71,7 +74,7 @@ rst_epilog = epilog_file.read()
 
 # General information about the project.
 project = 'Proxmox Backup'
-copyright = '2019-2020, Proxmox Support Team'
+copyright = '2019-2020, Proxmox Server Solutions GmbH'
 author = 'Proxmox Support Team'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -94,12 +97,10 @@ language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#
 # today = ''
 #
 # Else, today_fmt is used as the format for a strftime call.
-#
-# today_fmt = '%B %d, %Y'
+today_fmt = '%A, %d %B %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -145,7 +146,7 @@ pygments_style = 'sphinx'
 # keep_warnings = False
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = True
+todo_include_todos = not tags.has('release')
 
 
 # -- Options for HTML output ----------------------------------------------
@@ -153,13 +154,51 @@ todo_include_todos = True
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinxdoc'
+html_theme = 'alabaster'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-# html_theme_options = {}
+html_theme_options = {
+    'fixed_sidebar': True,
+    'sidebar_includehidden': False,
+    'sidebar_collapse': False,
+    'globaltoc_collapse': False,
+    'show_relbar_bottom': True,
+    'show_powered_by': False,
+
+    'extra_nav_links': {
+        'Proxmox Homepage': 'https://proxmox.com',
+        'PDF': 'proxmox-backup.pdf',
+    },
+
+    'sidebar_width': '320px',
+    'page_width': '1320px',
+    # font styles
+    'head_font_family': 'Lato, sans-serif',
+    'caption_font_family': 'Lato, sans-serif',
+    'caption_font_size': '20px',
+    'font_family': 'Open Sans, sans-serif',
+}
+
+# Alabaster theme recommends setting this fixed.
+# If you switch theme this needs to removed, probably.
+html_sidebars = {
+    '**': [
+        'sidebar-header.html',
+        'searchbox.html',
+        'navigation.html',
+        'relations.html',
+    ],
+
+    'index': [
+        'sidebar-header.html',
+        'searchbox.html',
+        'index-sidebar.html',
+    ]
+}
+
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -176,7 +215,7 @@ html_theme = 'sphinxdoc'
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
 #
-html_logo = 'images/proxmox-logo.svg'
+#html_logo = 'images/proxmox-logo.svg' # replaced by html_theme_options.logo
 
 # The name of an image file (relative to this directory) to use as a favicon of
 # the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
@@ -206,10 +245,6 @@ html_static_path = ['_static']
 #
 # html_use_smartypants = True
 
-# Custom sidebar templates, maps document names to template names.
-#
-# html_sidebars = {}
-
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
 #
@@ -229,7 +264,7 @@ html_static_path = ['_static']
 
 # If true, links to the reST sources are added to the pages.
 #
-# html_show_sourcelink = True
+html_show_sourcelink = False
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
 #

docs/custom.css

@@ -0,0 +1,52 @@
+div.sphinxsidebar {
+    height: calc(100% - 20px);
+    overflow: auto;
+}
+
+h1.logo-name {
+    font-size: 24px;
+}
+
+div.body img {
+    width: 250px;
+}
+pre {
+    padding: 5px 10px;
+}
+
+li a.current {
+    font-weight: bold;
+    border-bottom: 1px solid #000;
+}
+ul li.toctree-l1 {
+    margin-top: 0.5em;
+}
+ul li.toctree-l1 > a {
+    color: #000;
+}
+
+div.sphinxsidebar form.search {
+    margin-bottom: 5px;
+}
+
+div.sphinxsidebar h3 {
+    width: 100%;
+}
+
+div.sphinxsidebar h1.logo-name {
+    display: none;
+}
+@media screen and (max-width: 875px) {
+    div.sphinxsidebar p.logo {
+        display: initial;
+    }
+    div.sphinxsidebar h1.logo-name {
+        display: block;
+    }
+    div.sphinxsidebar span {
+        color: #AAA;
+    }
+    ul li.toctree-l1 > a {
+        color: #FFF;
+    }
+}

docs/epilog.rst

@@ -13,7 +13,7 @@
 .. _Proxmox: https://www.proxmox.com
 .. _Proxmox Community Forum: https://forum.proxmox.com
 .. _Proxmox Virtual Environment: https://www.proxmox.com/proxmox-ve
-  // FIXME
+.. FIXME
 .. _Proxmox Backup: https://pbs.proxmox.com/wiki/index.php/Main_Page
 .. _PBS Development List: https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
 .. _reStructuredText: https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html
@@ -38,3 +38,6 @@
 .. _RFC3399: https://tools.ietf.org/html/rfc3339
 .. _UTC: https://en.wikipedia.org/wiki/Coordinated_Universal_Time
 .. _ISO Week date: https://en.wikipedia.org/wiki/ISO_week_date
+
+.. _systemd.time manpage: https://manpages.debian.org/buster/systemd/systemd.time.7.en.html
+

docs/faq.rst

@@ -0,0 +1,71 @@
+FAQ
+===
+
+What distribution is Proxmox Backup Server (PBS) based on?
+----------------------------------------------------------
+
+Proxmox Backup Server is based on `Debian GNU/Linux <https://www.debian.org/>`_.
+
+
+Which platforms are supported as a backup source (client)?
+----------------------------------------------------------
+
+The client tool works on most modern Linux systems, meaning you are not limited
+to Debian-based backups.
+
+
+Will Proxmox Backup Server run on a 32-bit processor?
+-----------------------------------------------------
+
+Proxmox Backup Server only supports 64-bit CPUs (AMD or Intel). There are no
+future plans to support 32-bit processors.
+
+
+How long will my Proxmox Backup Server version be supported?
+------------------------------------------------------------
+
++-----------------------+--------------------+---------------+------------+--------------------+
+|Proxmox Backup Version | Debian Version     | First Release | Debian EOL | Proxmox Backup EOL |
++=======================+====================+===============+============+====================+
+|Proxmox Backup 1.x     | Debian 10 (Buster) | tba           | tba        | tba                |
++-----------------------+--------------------+---------------+------------+--------------------+
+
+
+Can I copy or synchronize my datastore to another location?
+-----------------------------------------------------------
+
+Proxmox Backup Server allows you to copy or synchroize datastores to other
+locations, through the use of *Remotes* and *Sync Jobs*. *Remote* is the term
+given to a separate server, which has a datastore that can be synced to a local store.
+A *Sync Job* is the process which is used to pull the contents of a datastore from
+a *Remote* to a local datastore.
+
+
+Can Proxmox Backup Server verify data integrity of a backup archive?
+--------------------------------------------------------------------
+
+Proxmox Backup Server uses a built-in SHA-256 checksum algorithm, to ensure
+data integrity. Within each backup, a manifest file (index.json) is created,
+which contains a list of all the backup files, along with their sizes and
+checksums. This manifest file is used to verify the integrity of each backup.
+
+
+When backing up to remote servers, do I have to trust the remote server?
+------------------------------------------------------------------------
+
+Proxmox Backup Server supports client-side encryption, meaning your data is
+encrypted before it reaches the server. Thus, in the event that an attacker
+gains access to the server, they will not be able to read the data.
+
+.. note:: Encryption is not enabled by default. To set up encryption, see the
+  `Encryption
+  <https://pbs.proxmox.com/docs/administration-guide.html#encryption>`_ section
+  of the Proxmox Backup Server Administration Guide.
+
+
+Is the backup incremental/deduplicated?
+---------------------------------------
+
+With Proxmox Backup Server, backups are sent incremental and data is
+deduplicated on the server.
+This minimizes both the storage consumed and the network impact.

docs/glossary.rst

@@ -51,14 +51,3 @@ Glossary
       A remote Proxmox Backup Server installation and credentials for a user on it.
       You can pull datastores from a remote to a local datastore in order to
       have redundant backups.
-
-   Schedule
-
-      Certain tasks, for example pruning and garbage collection, need to be
-      performed on a regular basis. Proxmox Backup Server uses a subset of the
-      `systemd Time and Date Specification
-      <https://www.freedesktop.org/software/systemd/man/systemd.time.html#>`_.
-      The subset currently supports time of day specifications and weekdays, in
-      addition to the shorthand expressions 'minutely', 'hourly', 'daily'.
-      There is no support for specifying timezones, the tasks are run in the
-      timezone configured on the server.

docs/index.rst

@@ -2,8 +2,8 @@
 
 Welcome to the Proxmox Backup documentation!
 ============================================
-
-Copyright (C) 2019-2020 Proxmox Server Solutions GmbH
+| Copyright (C) 2019-2020 Proxmox Server Solutions GmbH
+| Version |version| -- |today|
 
 Permission is granted to copy, distribute and/or modify this document under the
 terms of the GNU Free Documentation License, Version 1.3 or any later version
@@ -24,6 +24,7 @@ in the section entitled "GNU Free Documentation License".
    installation.rst
    administration-guide.rst
    sysadmin.rst
+   faq.rst
 
 .. raw:: latex
 
@@ -36,6 +37,7 @@ in the section entitled "GNU Free Documentation License".
    command-syntax.rst
    file-formats.rst
    backup-protocol.rst
+   calendarevents.rst
    glossary.rst
    GFDL.rst
 
@@ -43,10 +45,10 @@ in the section entitled "GNU Free Documentation License".
 
     .. toctree::
        :maxdepth: 2
+       :hidden:
        :caption: Developer Appendix
 
        todos.rst
 
 
-* :ref:`genindex`
-
+.. # * :ref:`genindex`

docs/introduction.rst

@@ -22,7 +22,7 @@ Architecture
 ------------
 
 Proxmox Backup Server uses a `client-server model`_. The server stores the
-backup data and provides an API to create and manage data stores. With the
+backup data and provides an API to create and manage datastores. With the
 API, it's also possible to manage disks and other server-side resources.
 
 The backup client uses this API to access the backed up data. With the command

docs/local-zfs.rst

@@ -1,3 +1,6 @@
+
+.. _chapter-zfs:
+
 ZFS on Linux
 ------------
 

examples/download-speed.rs

@@ -2,8 +2,6 @@ use std::io::Write;
 
 use anyhow::{Error};
 
-use chrono::{DateTime, Utc};
-
 use proxmox_backup::api2::types::Userid;
 use proxmox_backup::client::{HttpClient, HttpClientOptions, BackupReader};
 
@@ -34,9 +32,9 @@ async fn run() -> Result<(), Error> {
         .interactive(true)
         .ticket_cache(true);
 
-    let client = HttpClient::new(host, username, options)?;
+    let client = HttpClient::new(host, 8007, username, options)?;
 
-    let backup_time = "2019-06-28T10:49:48Z".parse::<DateTime<Utc>>()?;
+    let backup_time = proxmox::tools::time::parse_rfc3339("2019-06-28T10:49:48Z")?;
 
     let client = BackupReader::start(client, None, "store2", "host", "elsa", backup_time, true)
         .await?;

examples/upload-speed.rs

@@ -14,11 +14,11 @@ async fn upload_speed() -> Result<f64, Error> {
         .interactive(true)
         .ticket_cache(true);
 
-    let client = HttpClient::new(host, username, options)?;
+    let client = HttpClient::new(host, 8007, username, options)?;
 
-    let backup_time = chrono::Utc::now();
+    let backup_time = proxmox::tools::time::epoch_i64();
 
-    let client = BackupWriter::start(client, None, datastore, "host", "speedtest", backup_time, false).await?;
+    let client = BackupWriter::start(client, None, datastore, "host", "speedtest", backup_time, false, true).await?;
 
     println!("start upload speed test");
     let res = client.upload_speedtest(true).await?;

src/api2/access/acl.rs

@@ -175,7 +175,7 @@ pub fn update_acl(
     _rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<(), Error> {
 
-    let _lock = open_file_locked(acl::ACL_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(acl::ACL_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut tree, expected_digest) = acl::config()?;
 

src/api2/access/role.rs

@@ -14,7 +14,7 @@ use crate::config::acl::{Role, ROLE_NAMES, PRIVILEGES};
         type: Array,
         items: {
             type: Object,
-            description: "User name with description.",
+            description: "Role with description and privileges.",
             properties: {
                 roleid: {
                     type: Role,

src/api2/access/user.rs

@@ -8,6 +8,7 @@ use proxmox::tools::fs::open_file_locked;
 use crate::api2::types::*;
 use crate::config::user;
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_PERMISSIONS_MODIFY};
+use crate::config::cached_user_info::CachedUserInfo;
 
 pub const PBS_PASSWORD_SCHEMA: Schema = StringSchema::new("User Password.")
     .format(&PASSWORD_FORMAT)
@@ -25,10 +26,11 @@ pub const PBS_PASSWORD_SCHEMA: Schema = StringSchema::new("User Password.")
         items: { type: user::User },
     },
     access: {
-        permission: &Permission::Privilege(&["access", "users"], PRIV_SYS_AUDIT, false),
+        permission: &Permission::Anybody,
+        description: "Returns all or just the logged-in user, depending on privileges.",
     },
 )]
-/// List all users
+/// List users
 pub fn list_users(
     _param: Value,
     _info: &ApiMethod,
@@ -37,11 +39,21 @@ pub fn list_users(
 
     let (config, digest) = user::config()?;
 
-    let list = config.convert_to_typed_array("user")?;
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let top_level_privs = user_info.lookup_privs(&userid, &["access", "users"]);
+    let top_level_allowed = (top_level_privs & PRIV_SYS_AUDIT) != 0;
+
+    let filter_by_privs = |user: &user::User| {
+        top_level_allowed || user.userid == userid
+    };
+
+    let list:Vec<user::User> = config.convert_to_typed_array("user")?;
 
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
-    Ok(list)
+    Ok(list.into_iter().filter(filter_by_privs).collect())
 }
 
 #[api(
@@ -88,7 +100,7 @@ pub fn list_users(
 /// Create new user.
 pub fn create_user(password: Option<String>, param: Value) -> Result<(), Error> {
 
-    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let user: user::User = serde_json::from_value(param)?;
 
@@ -124,7 +136,10 @@ pub fn create_user(password: Option<String>, param: Value) -> Result<(), Error>
         type: user::User,
     },
     access: {
-        permission: &Permission::Privilege(&["access", "users"], PRIV_SYS_AUDIT, false),
+        permission: &Permission::Or(&[
+            &Permission::Privilege(&["access", "users"], PRIV_SYS_AUDIT, false),
+            &Permission::UserParam("userid"),
+        ]),
     },
 )]
 /// Read user configuration data.
@@ -177,7 +192,10 @@ pub fn read_user(userid: Userid, mut rpcenv: &mut dyn RpcEnvironment) -> Result<
         },
     },
     access: {
-        permission: &Permission::Privilege(&["access", "users"], PRIV_PERMISSIONS_MODIFY, false),
+        permission: &Permission::Or(&[
+            &Permission::Privilege(&["access", "users"], PRIV_PERMISSIONS_MODIFY, false),
+            &Permission::UserParam("userid"),
+        ]),
     },
 )]
 /// Update user configuration.
@@ -193,7 +211,7 @@ pub fn update_user(
     digest: Option<String>,
 ) -> Result<(), Error> {
 
-    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = user::config()?;
 
@@ -258,13 +276,16 @@ pub fn update_user(
         },
     },
     access: {
-        permission: &Permission::Privilege(&["access", "users"], PRIV_PERMISSIONS_MODIFY, false),
+        permission: &Permission::Or(&[
+            &Permission::Privilege(&["access", "users"], PRIV_PERMISSIONS_MODIFY, false),
+            &Permission::UserParam("userid"),
+        ]),
     },
 )]
 /// Remove a user from the configuration file.
 pub fn delete_user(userid: Userid, digest: Option<String>) -> Result<(), Error> {
 
-    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = user::config()?;
 

src/api2/admin/datastore.rs

@@ -1,6 +1,7 @@
 use std::collections::{HashSet, HashMap};
 use std::ffi::OsStr;
 use std::os::unix::ffi::OsStrExt;
+use std::sync::{Arc, Mutex};
 
 use anyhow::{bail, format_err, Error};
 use futures::*;
@@ -171,7 +172,7 @@ fn list_groups(
         let result_item = GroupListItem {
             backup_type: group.backup_type().to_string(),
             backup_id: group.backup_id().to_string(),
-            last_backup: info.backup_dir.backup_time().timestamp(),
+            last_backup: info.backup_dir.backup_time(),
             backup_count: list.len() as u64,
             files: info.files.clone(),
             owner: Some(owner),
@@ -229,7 +230,7 @@ pub fn list_snapshot_files(
 
     let datastore = DataStore::lookup_datastore(&store)?;
 
-    let snapshot = BackupDir::new(backup_type, backup_id, backup_time);
+    let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
 
     let allowed = (user_privs & (PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_READ)) != 0;
     if !allowed { check_backup_owner(&datastore, snapshot.group(), &userid)?; }
@@ -279,7 +280,7 @@ fn delete_snapshot(
     let user_info = CachedUserInfo::new()?;
     let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
 
-    let snapshot = BackupDir::new(backup_type, backup_id, backup_time);
+    let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
 
     let datastore = DataStore::lookup_datastore(&store)?;
 
@@ -402,7 +403,7 @@ pub fn list_snapshots (
         let result_item = SnapshotListItem {
             backup_type: group.backup_type().to_string(),
             backup_id: group.backup_id().to_string(),
-            backup_time: info.backup_dir.backup_time().timestamp(),
+            backup_time: info.backup_dir.backup_time(),
             comment,
             verification,
             files,
@@ -489,7 +490,7 @@ pub fn verify(
     match (backup_type, backup_id, backup_time) {
         (Some(backup_type), Some(backup_id), Some(backup_time)) => {
             worker_id = format!("{}_{}_{}_{:08X}", store, backup_type, backup_id, backup_time);
-            let dir = BackupDir::new(backup_type, backup_id, backup_time);
+            let dir = BackupDir::new(backup_type, backup_id, backup_time)?;
             backup_dir = Some(dir);
         }
         (Some(backup_type), Some(backup_id), None) => {
@@ -512,18 +513,27 @@ pub fn verify(
         userid,
         to_stdout,
         move |worker| {
+            let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*16)));
+            let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
+
             let failed_dirs = if let Some(backup_dir) = backup_dir {
-                let mut verified_chunks = HashSet::with_capacity(1024*16);
-                let mut corrupt_chunks = HashSet::with_capacity(64);
                 let mut res = Vec::new();
-                if !verify_backup_dir(&datastore, &backup_dir, &mut verified_chunks, &mut corrupt_chunks, &worker)? {
+                if !verify_backup_dir(datastore, &backup_dir, verified_chunks, corrupt_chunks, worker.clone())? {
                     res.push(backup_dir.to_string());
                 }
                 res
             } else if let Some(backup_group) = backup_group {
-                verify_backup_group(&datastore, &backup_group, &worker)?
+                let (_count, failed_dirs) = verify_backup_group(
+                    datastore,
+                    &backup_group,
+                    verified_chunks,
+                    corrupt_chunks,
+                    None,
+                    worker.clone(),
+                )?;
+                failed_dirs
             } else {
-                verify_all_backups(&datastore, &worker)?
+                verify_all_backups(datastore, worker.clone())?
             };
             if failed_dirs.len() > 0 {
                 worker.log("Failed to verify following snapshots:");
@@ -663,7 +673,7 @@ fn prune(
             prune_result.push(json!({
                 "backup-type": group.backup_type(),
                 "backup-id": group.backup_id(),
-                "backup-time": backup_time.timestamp(),
+                "backup-time": backup_time,
                 "keep": keep,
             }));
         }
@@ -687,7 +697,7 @@ fn prune(
             if keep_all { keep = true; }
 
             let backup_time = info.backup_dir.backup_time();
-            let timestamp = BackupDir::backup_time_to_string(backup_time);
+            let timestamp = info.backup_dir.backup_time_string();
             let group = info.backup_dir.group();
 
 
@@ -704,7 +714,7 @@ fn prune(
             prune_result.push(json!({
                 "backup-type": group.backup_type(),
                 "backup-id": group.backup_id(),
-                "backup-time": backup_time.timestamp(),
+                "backup-time": backup_time,
                 "keep": keep,
             }));
 
@@ -887,7 +897,7 @@ fn download_file(
         let backup_id = tools::required_string_param(&param, "backup-id")?;
         let backup_time = tools::required_integer_param(&param, "backup-time")?;
 
-        let backup_dir = BackupDir::new(backup_type, backup_id, backup_time);
+        let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
         let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
         if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
@@ -960,7 +970,7 @@ fn download_file_decoded(
         let backup_id = tools::required_string_param(&param, "backup-id")?;
         let backup_time = tools::required_integer_param(&param, "backup-time")?;
 
-        let backup_dir = BackupDir::new(backup_type, backup_id, backup_time);
+        let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
         let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
         if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
@@ -1073,7 +1083,7 @@ fn upload_backup_log(
         let backup_id = tools::required_string_param(&param, "backup-id")?;
         let backup_time = tools::required_integer_param(&param, "backup-time")?;
 
-        let backup_dir = BackupDir::new(backup_type, backup_id, backup_time);
+        let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
         let userid: Userid = rpcenv.get_user().unwrap().parse()?;
         check_backup_owner(&datastore, backup_dir.group(), &userid)?;
@@ -1087,7 +1097,7 @@ fn upload_backup_log(
         }
 
         println!("Upload backup log to {}/{}/{}/{}/{}", store,
-                 backup_type, backup_id, BackupDir::backup_time_to_string(backup_dir.backup_time()), file_name);
+                 backup_type, backup_id, backup_dir.backup_time_string(), file_name);
 
         let data = req_body
             .map_err(Error::from)
@@ -1149,7 +1159,7 @@ fn catalog(
     let user_info = CachedUserInfo::new()?;
     let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
 
-    let backup_dir = BackupDir::new(backup_type, backup_id, backup_time);
+    let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
     let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
     if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
@@ -1266,7 +1276,7 @@ fn pxar_file_download(
         let backup_id = tools::required_string_param(&param, "backup-id")?;
         let backup_time = tools::required_integer_param(&param, "backup-time")?;
 
-        let backup_dir = BackupDir::new(backup_type, backup_id, backup_time);
+        let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
         let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
         if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
@@ -1407,7 +1417,7 @@ fn get_notes(
     let user_info = CachedUserInfo::new()?;
     let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
 
-    let backup_dir = BackupDir::new(backup_type, backup_id, backup_time);
+    let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
     let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
     if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
@@ -1460,7 +1470,7 @@ fn set_notes(
     let user_info = CachedUserInfo::new()?;
     let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
 
-    let backup_dir = BackupDir::new(backup_type, backup_id, backup_time);
+    let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
     let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
     if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }

src/api2/admin/sync.rs

@@ -57,7 +57,8 @@ pub fn list_sync_jobs(
         job.next_run = (|| -> Option<i64> {
             let schedule = job.schedule.as_ref()?;
             let event = parse_calendar_event(&schedule).ok()?;
-            compute_next_event(&event, last, false).ok()
+            // ignore errors
+            compute_next_event(&event, last, false).unwrap_or_else(|_| None)
         })();
     }
 

src/api2/backup.rs

@@ -38,6 +38,7 @@ pub const API_METHOD_UPGRADE_BACKUP: ApiMethod = ApiMethod::new(
             ("backup-id", false, &BACKUP_ID_SCHEMA),
             ("backup-time", false, &BACKUP_TIME_SCHEMA),
             ("debug", true, &BooleanSchema::new("Enable verbose debug logging.").schema()),
+            ("benchmark", true, &BooleanSchema::new("Job is a benchmark (do not keep data).").schema()),
         ]),
     )
 ).access(
@@ -56,6 +57,7 @@ fn upgrade_to_backup_protocol(
 
 async move {
     let debug = param["debug"].as_bool().unwrap_or(false);
+    let benchmark = param["benchmark"].as_bool().unwrap_or(false);
 
     let userid: Userid = rpcenv.get_user().unwrap().parse()?;
 
@@ -90,16 +92,50 @@ async move {
 
     let backup_group = BackupGroup::new(backup_type, backup_id);
 
+    let worker_type = if backup_type == "host" && backup_id == "benchmark" {
+        if !benchmark {
+            bail!("unable to run benchmark without --benchmark flags");
+        }
+        "benchmark"
+    } else {
+        if benchmark {
+            bail!("benchmark flags is only allowed on 'host/benchmark'");
+        }
+        "backup"
+    };
+
     // lock backup group to only allow one backup per group at a time
     let (owner, _group_guard) = datastore.create_locked_backup_group(&backup_group, &userid)?;
 
     // permission check
-    if owner != userid { // only the owner is allowed to create additional snapshots
+    if owner != userid && worker_type != "benchmark" {
+        // only the owner is allowed to create additional snapshots
         bail!("backup owner check failed ({} != {})", userid, owner);
     }
 
-    let last_backup = BackupInfo::last_backup(&datastore.base_path(), &backup_group, true).unwrap_or(None);
-    let backup_dir = BackupDir::new_with_group(backup_group.clone(), backup_time);
+    let last_backup = {
+        let info = BackupInfo::last_backup(&datastore.base_path(), &backup_group, true).unwrap_or(None);
+        if let Some(info) = info {
+            let (manifest, _) = datastore.load_manifest(&info.backup_dir)?;
+            let verify = manifest.unprotected["verify_state"].clone();
+            match serde_json::from_value::<SnapshotVerifyState>(verify) {
+                Ok(verify) => {
+                    match verify.state {
+                        VerifyState::Ok => Some(info),
+                        VerifyState::Failed => None,
+                    }
+                },
+                Err(_) => {
+                    // no verify state found, treat as valid
+                    Some(info)
+                }
+            }
+        } else {
+            None
+        }
+    };
+
+    let backup_dir = BackupDir::with_group(backup_group.clone(), backup_time)?;
 
     let _last_guard = if let Some(last) = &last_backup {
         if backup_dir.backup_time() <= last.backup_dir.backup_time() {
@@ -116,14 +152,15 @@ async move {
     let (path, is_new, _snap_guard) = datastore.create_locked_backup_dir(&backup_dir)?;
     if !is_new { bail!("backup directory already exists."); }
 
-    WorkerTask::spawn("backup", Some(worker_id), userid.clone(), true, move |worker| {
+
+    WorkerTask::spawn(worker_type, Some(worker_id), userid.clone(), true, move |worker| {
         let mut env = BackupEnvironment::new(
             env_type, userid, worker.clone(), datastore, backup_dir);
 
         env.debug = debug;
         env.last_backup = last_backup;
 
-        env.log(format!("starting new backup on datastore '{}': {:?}", store, path));
+        env.log(format!("starting new {} on datastore '{}': {:?}", worker_type, store, path));
 
         let service = H2Service::new(env.clone(), worker.clone(), &BACKUP_API_ROUTER, debug);
 
@@ -143,6 +180,7 @@ async move {
                 let window_size = 32*1024*1024; // max = (1 << 31) - 2
                 http.http2_initial_stream_window_size(window_size);
                 http.http2_initial_connection_window_size(window_size);
+                http.http2_max_frame_size(4*1024*1024);
 
                 http.serve_connection(conn, service)
                     .map_err(Error::from)
@@ -160,7 +198,11 @@ async move {
                 req = req_fut => req,
                 abrt = abort_future => abrt,
             };
-
+            if benchmark {
+                env.log("benchmark finished successfully");
+                tools::runtime::block_in_place(|| env.remove_backup())?;
+                return Ok(());
+            }
             match (res, env.ensure_finished()) {
                 (Ok(_), Ok(())) => {
                     env.log("backup finished successfully");
@@ -180,7 +222,7 @@ async move {
                 (Err(err), Err(_)) => {
                     env.log(format!("backup failed: {}", err));
                     env.log("removing failed backup");
-                    env.remove_backup()?;
+                    tools::runtime::block_in_place(|| env.remove_backup())?;
                     Err(err)
                 },
             }
@@ -334,7 +376,7 @@ fn create_fixed_index(
         let last_backup = match &env.last_backup {
             Some(info) => info,
             None => {
-                bail!("cannot reuse index - no previous backup exists");
+                bail!("cannot reuse index - no valid previous backup exists");
             }
         };
 
@@ -649,7 +691,7 @@ fn download_previous(
 
         let last_backup = match &env.last_backup {
             Some(info) => info,
-            None => bail!("no previous backup"),
+            None => bail!("no valid previous backup"),
         };
 
         let mut path = env.datastore.snapshot_path(&last_backup.backup_dir);

src/api2/backup/environment.rs

@@ -66,13 +66,16 @@ struct FixedWriterState {
     incremental: bool,
 }
 
+// key=digest, value=length
+type KnownChunksMap = HashMap<[u8;32], u32>;
+
 struct SharedBackupState {
     finished: bool,
     uid_counter: usize,
     file_counter: usize, // successfully uploaded files
     dynamic_writers: HashMap<usize, DynamicWriterState>,
     fixed_writers: HashMap<usize, FixedWriterState>,
-    known_chunks: HashMap<[u8;32], u32>,
+    known_chunks: KnownChunksMap,
     backup_size: u64, // sums up size of all files
     backup_stat: UploadStatistic,
 }
@@ -457,11 +460,11 @@ impl BackupEnvironment {
     /// Mark backup as finished
     pub fn finish_backup(&self) -> Result<(), Error> {
         let mut state = self.state.lock().unwrap();
-        // test if all writer are correctly closed
 
         state.ensure_unfinished()?;
 
-        if state.dynamic_writers.len() != 0 {
+        // test if all writer are correctly closed
+        if state.dynamic_writers.len() != 0 || state.fixed_writers.len() != 0 {
             bail!("found open index writer - unable to finish backup");
         }
 

src/api2/backup/upload_chunk.rs

@@ -61,12 +61,15 @@ impl Future for UploadChunk {
                         let (is_duplicate, compressed_size) = match proxmox::try_block! {
                             let mut chunk = DataBlob::from_raw(raw_data)?;
 
-                            chunk.verify_unencrypted(this.size as usize, &this.digest)?;
+                            tools::runtime::block_in_place(|| {
+                                chunk.verify_unencrypted(this.size as usize, &this.digest)?;
 
-                            // always comput CRC at server side
-                            chunk.set_crc(chunk.compute_crc());
+                                // always comput CRC at server side
+                                chunk.set_crc(chunk.compute_crc());
+
+                                this.store.insert_chunk(&chunk, &this.digest)
+                            })
 
-                            this.store.insert_chunk(&chunk, &this.digest)
                         } {
                             Ok(res) => res,
                             Err(err) => break err,

src/api2/config/datastore.rs

@@ -9,6 +9,7 @@ use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
 use crate::backup::*;
+use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::datastore::{self, DataStoreConfig, DIR_NAME_SCHEMA};
 use crate::config::acl::{PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY};
 
@@ -22,7 +23,7 @@ use crate::config::acl::{PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY};
         items: { type: datastore::DataStoreConfig },
     },
     access: {
-        permission: &Permission::Privilege(&["datastore"], PRIV_DATASTORE_AUDIT, false),
+        permission: &Permission::Anybody,
     },
 )]
 /// List all datastores
@@ -33,11 +34,18 @@ pub fn list_datastores(
 
     let (config, digest) = datastore::config()?;
 
-    let list = config.convert_to_typed_array("datastore")?;
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
-    Ok(list)
+    let list:Vec<DataStoreConfig> = config.convert_to_typed_array("datastore")?;
+    let filter_by_privs = |store: &DataStoreConfig| {
+        let user_privs = user_info.lookup_privs(&userid, &["datastore", &store.name]);
+        (user_privs & PRIV_DATASTORE_AUDIT) != 0
+    };
+
+    Ok(list.into_iter().filter(filter_by_privs).collect())
 }
 
 
@@ -67,6 +75,10 @@ pub fn list_datastores(
                 optional: true,
                 schema: PRUNE_SCHEDULE_SCHEMA,
             },
+            "verify-schedule": {
+                optional: true,
+                schema: VERIFY_SCHEDULE_SCHEMA,
+            },
             "keep-last": {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -100,7 +112,7 @@ pub fn list_datastores(
 /// Create new datastore config.
 pub fn create_datastore(param: Value) -> Result<(), Error> {
 
-    let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let datastore: datastore::DataStoreConfig = serde_json::from_value(param.clone())?;
 
@@ -119,6 +131,10 @@ pub fn create_datastore(param: Value) -> Result<(), Error> {
 
     datastore::save_config(&config)?;
 
+    crate::config::jobstate::create_state_file("prune", &datastore.name)?;
+    crate::config::jobstate::create_state_file("garbage_collection", &datastore.name)?;
+    crate::config::jobstate::create_state_file("verify", &datastore.name)?;
+
     Ok(())
 }
 
@@ -163,6 +179,8 @@ pub enum DeletableProperty {
     gc_schedule,
     /// Delete the prune job schedule.
     prune_schedule,
+    /// Delete the verify schedule property
+    verify_schedule,
     /// Delete the keep-last property
     keep_last,
     /// Delete the keep-hourly property
@@ -196,6 +214,10 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: PRUNE_SCHEDULE_SCHEMA,
             },
+            "verify-schedule": {
+                optional: true,
+                schema: VERIFY_SCHEDULE_SCHEMA,
+            },
             "keep-last": {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -244,6 +266,7 @@ pub fn update_datastore(
     comment: Option<String>,
     gc_schedule: Option<String>,
     prune_schedule: Option<String>,
+    verify_schedule: Option<String>,
     keep_last: Option<u64>,
     keep_hourly: Option<u64>,
     keep_daily: Option<u64>,
@@ -254,7 +277,7 @@ pub fn update_datastore(
     digest: Option<String>,
 ) -> Result<(), Error> {
 
-    let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     // pass/compare digest
     let (mut config, expected_digest) = datastore::config()?;
@@ -272,6 +295,7 @@ pub fn update_datastore(
                 DeletableProperty::comment => { data.comment = None; },
                 DeletableProperty::gc_schedule => { data.gc_schedule = None; },
                 DeletableProperty::prune_schedule => { data.prune_schedule = None; },
+                DeletableProperty::verify_schedule => { data.verify_schedule = None; },
                 DeletableProperty::keep_last => { data.keep_last = None; },
                 DeletableProperty::keep_hourly => { data.keep_hourly = None; },
                 DeletableProperty::keep_daily => { data.keep_daily = None; },
@@ -291,8 +315,23 @@ pub fn update_datastore(
         }
     }
 
-    if gc_schedule.is_some() { data.gc_schedule = gc_schedule; }
-    if prune_schedule.is_some() { data.prune_schedule = prune_schedule; }
+    let mut gc_schedule_changed = false;
+    if gc_schedule.is_some() {
+        gc_schedule_changed = data.gc_schedule != gc_schedule;
+        data.gc_schedule = gc_schedule;
+    }
+
+    let mut prune_schedule_changed = false;
+    if prune_schedule.is_some() {
+        prune_schedule_changed = data.prune_schedule != prune_schedule;
+        data.prune_schedule = prune_schedule;
+    }
+
+    let mut verify_schedule_changed = false;
+    if verify_schedule.is_some() {
+        verify_schedule_changed = data.verify_schedule != verify_schedule;
+        data.verify_schedule = verify_schedule;
+    }
 
     if keep_last.is_some() { data.keep_last = keep_last; }
     if keep_hourly.is_some() { data.keep_hourly = keep_hourly; }
@@ -305,6 +344,20 @@ pub fn update_datastore(
 
     datastore::save_config(&config)?;
 
+    // we want to reset the statefiles, to avoid an immediate action in some cases
+    // (e.g. going from monthly to weekly in the second week of the month)
+    if gc_schedule_changed {
+        crate::config::jobstate::create_state_file("garbage_collection", &name)?;
+    }
+
+    if prune_schedule_changed {
+        crate::config::jobstate::create_state_file("prune", &name)?;
+    }
+
+    if verify_schedule_changed {
+        crate::config::jobstate::create_state_file("verify", &name)?;
+    }
+
     Ok(())
 }
 
@@ -328,7 +381,7 @@ pub fn update_datastore(
 /// Remove a datastore configuration.
 pub fn delete_datastore(name: String, digest: Option<String>) -> Result<(), Error> {
 
-    let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = datastore::config()?;
 
@@ -344,6 +397,11 @@ pub fn delete_datastore(name: String, digest: Option<String>) -> Result<(), Erro
 
     datastore::save_config(&config)?;
 
+    // ignore errors
+    let _ = crate::config::jobstate::remove_state_file("prune", &name);
+    let _ = crate::config::jobstate::remove_state_file("garbage_collection", &name);
+    let _ = crate::config::jobstate::remove_state_file("verify", &name);
+
     Ok(())
 }
 

src/api2/config/remote.rs

@@ -60,6 +60,12 @@ pub fn list_remotes(
             host: {
                 schema: DNS_NAME_OR_IP_SCHEMA,
             },
+            port: {
+                description: "The (optional) port.",
+                type: u16,
+                optional: true,
+                default: 8007,
+            },
             userid: {
                 type: Userid,
             },
@@ -79,7 +85,7 @@ pub fn list_remotes(
 /// Create new remote.
 pub fn create_remote(password: String, param: Value) -> Result<(), Error> {
 
-    let _lock = open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let mut data = param.clone();
     data["password"] = Value::from(base64::encode(password.as_bytes()));
@@ -136,6 +142,8 @@ pub enum DeletableProperty {
     comment,
     /// Delete the fingerprint property.
     fingerprint,
+    /// Delete the port property.
+    port,
 }
 
 #[api(
@@ -153,6 +161,11 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: DNS_NAME_OR_IP_SCHEMA,
             },
+            port: {
+                description: "The (optional) port.",
+                type: u16,
+                optional: true,
+            },
             userid: {
                 optional: true,
                 type: Userid,
@@ -188,6 +201,7 @@ pub fn update_remote(
     name: String,
     comment: Option<String>,
     host: Option<String>,
+    port: Option<u16>,
     userid: Option<Userid>,
     password: Option<String>,
     fingerprint: Option<String>,
@@ -195,7 +209,7 @@ pub fn update_remote(
     digest: Option<String>,
 ) -> Result<(), Error> {
 
-    let _lock = open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = remote::config()?;
 
@@ -211,6 +225,7 @@ pub fn update_remote(
             match delete_prop {
                 DeletableProperty::comment => { data.comment = None; },
                 DeletableProperty::fingerprint => { data.fingerprint = None; },
+                DeletableProperty::port => { data.port = None; },
             }
         }
     }
@@ -224,6 +239,7 @@ pub fn update_remote(
         }
     }
     if let Some(host) = host { data.host = host; }
+    if port.is_some() { data.port = port; }
     if let Some(userid) = userid { data.userid = userid; }
     if let Some(password) = password { data.password = password; }
 
@@ -256,7 +272,7 @@ pub fn update_remote(
 /// Remove a remote from the configuration file.
 pub fn delete_remote(name: String, digest: Option<String>) -> Result<(), Error> {
 
-    let _lock = open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = remote::config()?;
 

src/api2/config/sync.rs

@@ -69,7 +69,7 @@ pub fn list_sync_jobs(
 /// Create a new sync job.
 pub fn create_sync_job(param: Value) -> Result<(), Error> {
 
-    let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let sync_job: sync::SyncJobConfig = serde_json::from_value(param.clone())?;
 
@@ -187,7 +187,7 @@ pub fn update_sync_job(
     digest: Option<String>,
 ) -> Result<(), Error> {
 
-    let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     // pass/compare digest
     let (mut config, expected_digest) = sync::config()?;
@@ -250,7 +250,7 @@ pub fn update_sync_job(
 /// Remove a sync job configuration
 pub fn delete_sync_job(id: String, digest: Option<String>) -> Result<(), Error> {
 
-    let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = sync::config()?;
 

src/api2/node/disks/zfs.rs

@@ -25,6 +25,8 @@ use crate::server::WorkerTask;
 
 use crate::api2::types::*;
 
+use crate::tools::systemd;
+
 pub const DISK_ARRAY_SCHEMA: Schema = ArraySchema::new(
     "Disk name list.", &BLOCKDEVICE_NAME_SCHEMA)
     .schema();
@@ -355,6 +357,11 @@ pub fn create_zpool(
             let output = crate::tools::run_command(command, None)?;
             worker.log(output);
 
+            if std::path::Path::new("/lib/systemd/system/zfs-import@.service").exists() {
+                let import_unit = format!("zfs-import@{}.service", systemd::escape_unit(&name, false));
+                systemd::enable_unit(&import_unit)?;
+            }
+
             if let Some(compression) = compression {
                 let mut command = std::process::Command::new("zfs");
                 command.args(&["set", &format!("compression={}", compression), &name]);

src/api2/node/network.rs

@@ -198,6 +198,14 @@ pub fn read_interface(iface: String) -> Result<Value, Error> {
                 type: LinuxBondMode,
                 optional: true,
             },
+            "bond-primary": {
+                schema: NETWORK_INTERFACE_NAME_SCHEMA,
+                optional: true,
+            },
+            bond_xmit_hash_policy: {
+                type: BondXmitHashPolicy,
+                optional: true,
+            },
             slaves: {
                 schema: NETWORK_INTERFACE_LIST_SCHEMA,
                 optional: true,
@@ -224,6 +232,8 @@ pub fn create_interface(
     bridge_ports: Option<String>,
     bridge_vlan_aware: Option<bool>,
     bond_mode: Option<LinuxBondMode>,
+    bond_primary: Option<String>,
+    bond_xmit_hash_policy: Option<BondXmitHashPolicy>,
     slaves: Option<String>,
     param: Value,
 ) -> Result<(), Error> {
@@ -231,7 +241,7 @@ pub fn create_interface(
     let interface_type = crate::tools::required_string_param(&param, "type")?;
     let interface_type: NetworkInterfaceType = serde_json::from_value(interface_type.into())?;
 
-    let _lock = open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, _digest) = network::config()?;
 
@@ -284,7 +294,23 @@ pub fn create_interface(
             if bridge_vlan_aware.is_some() { interface.bridge_vlan_aware = bridge_vlan_aware; }
         }
         NetworkInterfaceType::Bond => {
-            if bond_mode.is_some() { interface.bond_mode = bond_mode; }
+            if let Some(mode) = bond_mode {
+                interface.bond_mode = bond_mode;
+                if bond_primary.is_some() {
+                    if mode != LinuxBondMode::active_backup {
+                        bail!("bond-primary is only valid with Active/Backup mode");
+                    }
+                    interface.bond_primary = bond_primary;
+                }
+                if bond_xmit_hash_policy.is_some() {
+                    if mode != LinuxBondMode::ieee802_3ad &&
+                       mode != LinuxBondMode::balance_xor
+                    {
+                        bail!("bond_xmit_hash_policy is only valid with LACP(802.3ad) or balance-xor mode");
+                    }
+                    interface.bond_xmit_hash_policy = bond_xmit_hash_policy;
+                }
+            }
             if let Some(slaves) = slaves {
                 let slaves = split_interface_list(&slaves)?;
                 interface.set_bond_slaves(slaves)?;
@@ -343,6 +369,11 @@ pub enum DeletableProperty {
     bridge_vlan_aware,
     /// Delete bond-slaves (set to 'none')
     slaves,
+    /// Delete bond-primary
+    #[serde(rename = "bond-primary")]
+    bond_primary,
+    /// Delete bond transmit hash policy
+    bond_xmit_hash_policy,
 }
 
 
@@ -420,6 +451,14 @@ pub enum DeletableProperty {
                 type: LinuxBondMode,
                 optional: true,
             },
+            "bond-primary": {
+                schema: NETWORK_INTERFACE_NAME_SCHEMA,
+                optional: true,
+            },
+            bond_xmit_hash_policy: {
+                type: BondXmitHashPolicy,
+                optional: true,
+            },
             slaves: {
                 schema: NETWORK_INTERFACE_LIST_SCHEMA,
                 optional: true,
@@ -458,13 +497,15 @@ pub fn update_interface(
     bridge_ports: Option<String>,
     bridge_vlan_aware: Option<bool>,
     bond_mode: Option<LinuxBondMode>,
+    bond_primary: Option<String>,
+    bond_xmit_hash_policy: Option<BondXmitHashPolicy>,
     slaves: Option<String>,
     delete: Option<Vec<DeletableProperty>>,
     digest: Option<String>,
     param: Value,
 ) -> Result<(), Error> {
 
-    let _lock = open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = network::config()?;
 
@@ -501,6 +542,8 @@ pub fn update_interface(
                 DeletableProperty::bridge_ports => { interface.set_bridge_ports(Vec::new())?; }
                 DeletableProperty::bridge_vlan_aware => { interface.bridge_vlan_aware = None; }
                 DeletableProperty::slaves => { interface.set_bond_slaves(Vec::new())?; }
+                DeletableProperty::bond_primary => { interface.bond_primary = None; }
+                DeletableProperty::bond_xmit_hash_policy => { interface.bond_xmit_hash_policy = None }
             }
         }
     }
@@ -518,7 +561,23 @@ pub fn update_interface(
         let slaves = split_interface_list(&slaves)?;
         interface.set_bond_slaves(slaves)?;
     }
-    if bond_mode.is_some() { interface.bond_mode = bond_mode; }
+    if let Some(mode) = bond_mode {
+        interface.bond_mode = bond_mode;
+        if bond_primary.is_some() {
+            if mode != LinuxBondMode::active_backup {
+                bail!("bond-primary is only valid with Active/Backup mode");
+            }
+            interface.bond_primary = bond_primary;
+        }
+        if bond_xmit_hash_policy.is_some() {
+            if mode != LinuxBondMode::ieee802_3ad &&
+               mode != LinuxBondMode::balance_xor
+            {
+                bail!("bond_xmit_hash_policy is only valid with LACP(802.3ad) or balance-xor mode");
+            }
+            interface.bond_xmit_hash_policy = bond_xmit_hash_policy;
+        }
+    }
 
     if let Some(cidr) = cidr {
         let (_, _, is_v6) = network::parse_cidr(&cidr)?;
@@ -587,7 +646,7 @@ pub fn update_interface(
 /// Remove network interface configuration.
 pub fn delete_interface(iface: String, digest: Option<String>) -> Result<(), Error> {
 
-    let _lock = open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = network::config()?;
 

src/api2/node/rrd.rs

@@ -1,10 +1,10 @@
 use anyhow::Error;
 use serde_json::{Value, json};
 
-use proxmox::api::{api, Router};
+use proxmox::api::{api, Permission, Router};
 
 use crate::api2::types::*;
-use crate::tools::epoch_now_f64;
+use crate::config::acl::PRIV_SYS_AUDIT;
 use crate::rrd::{extract_cached_data, RRD_DATA_ENTRIES};
 
 pub fn create_value_from_rrd(
@@ -15,7 +15,7 @@ pub fn create_value_from_rrd(
 ) -> Result<Value, Error> {
 
     let mut result = Vec::new();
-    let now = epoch_now_f64()?;
+    let now = proxmox::tools::time::epoch_f64();
 
     for name in list {
         let (start, reso, list) = match extract_cached_data(basedir, name, now, timeframe, cf) {
@@ -57,6 +57,9 @@ pub fn create_value_from_rrd(
             },
         },
     },
+    access: {
+        permission: &Permission::Privilege(&["system", "status"], PRIV_SYS_AUDIT, false),
+    },
 )]
 /// Read node stats
 fn get_node_stats(

src/api2/node/subscription.rs

@@ -1,11 +1,12 @@
 use anyhow::{Error};
 use serde_json::{json, Value};
 
-use proxmox::api::{api, Router, Permission};
+use proxmox::api::{api, Router, RpcEnvironment, Permission};
 
 use crate::tools;
 use crate::config::acl::PRIV_SYS_AUDIT;
-use crate::api2::types::NODE_SCHEMA;
+use crate::config::cached_user_info::CachedUserInfo;
+use crate::api2::types::{NODE_SCHEMA, Userid};
 
 #[api(
     input: {
@@ -28,7 +29,7 @@ use crate::api2::types::NODE_SCHEMA;
             },
             serverid: {
                 type: String,
-                description: "The unique server ID.",
+                description: "The unique server ID, if permitted to access.",
             },
             url: {
                 type: String,
@@ -37,18 +38,29 @@ use crate::api2::types::NODE_SCHEMA;
         },
     },
     access: {
-        permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
+        permission: &Permission::Anybody,
     },
 )]
 /// Read subscription info.
-fn get_subscription(_param: Value) -> Result<Value, Error> {
+fn get_subscription(
+    _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Value, Error> {
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    let user_privs = user_info.lookup_privs(&userid, &[]);
+    let server_id = if (user_privs & PRIV_SYS_AUDIT) != 0 {
+        tools::get_hardware_address()?
+    } else {
+        "hidden".to_string()
+    };
 
     let url = "https://www.proxmox.com/en/proxmox-backup-server/pricing";
     Ok(json!({
         "status": "NotFound",
-	"message": "There is no subscription key",
-	"serverid": tools::get_hardware_address()?,
-	"url":  url,
+        "message": "There is no subscription key",
+        "serverid": server_id,
+        "url":  url,
      }))
 }
 

src/api2/node/tasks.rs

@@ -10,7 +10,7 @@ use proxmox::{identity, list_subdirs_api_method, sortable};
 
 use crate::tools;
 use crate::api2::types::*;
-use crate::server::{self, UPID, TaskState};
+use crate::server::{self, UPID, TaskState, TaskListInfoIterator};
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
 use crate::config::cached_user_info::CachedUserInfo;
 
@@ -303,6 +303,7 @@ pub fn list_tasks(
     limit: u64,
     errors: bool,
     running: bool,
+    userfilter: Option<String>,
     param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<TaskListItem>, Error> {
@@ -315,57 +316,55 @@ pub fn list_tasks(
 
     let store = param["store"].as_str();
 
-    let userfilter = param["userfilter"].as_str();
+    let list = TaskListInfoIterator::new(running)?;
 
-    let list = server::read_task_list()?;
+    let result: Vec<TaskListItem> = list
+        .take_while(|info| !info.is_err())
+        .filter_map(|info| {
+        let info = match info {
+            Ok(info) => info,
+            Err(_) => return None,
+        };
 
-    let mut result = vec![];
+        if !list_all && info.upid.userid != userid { return None; }
 
-    let mut count = 0;
-
-    for info in list {
-        if !list_all && info.upid.userid != userid { continue; }
-
-
-        if let Some(userid) = userfilter {
-            if !info.upid.userid.as_str().contains(userid) { continue; }
+        if let Some(userid) = &userfilter {
+            if !info.upid.userid.as_str().contains(userid) { return None; }
         }
 
         if let Some(store) = store {
             // Note: useful to select all tasks spawned by proxmox-backup-client
             let worker_id = match &info.upid.worker_id {
                 Some(w) => w,
-                None => continue, // skip
+                None => return None, // skip
             };
 
             if info.upid.worker_type == "backup" || info.upid.worker_type == "restore" ||
                 info.upid.worker_type == "prune"
             {
                 let prefix = format!("{}_", store);
-                if !worker_id.starts_with(&prefix) { continue; }
+                if !worker_id.starts_with(&prefix) { return None; }
             } else if info.upid.worker_type == "garbage_collection" {
-                if worker_id != store { continue; }
+                if worker_id != store { return None; }
             } else {
-                continue; // skip
+                return None; // skip
             }
         }
 
-        if let Some(ref state) = info.state {
-            if running { continue; }
-            match state {
-                crate::server::TaskState::OK { .. } if errors => continue,
-                _ => {},
-            }
+        match info.state {
+            Some(_) if running => return None,
+            Some(crate::server::TaskState::OK { .. }) if errors => return None,
+            _ => {},
         }
 
-        if (count as u64) < start {
-            count += 1;
-            continue;
-        } else {
-            count += 1;
-        }
+        Some(info.into())
+    }).skip(start as usize)
+        .take(limit as usize)
+        .collect();
 
-        if (result.len() as u64) < limit { result.push(info.into()); };
+    let mut count = result.len() + start as usize;
+    if result.len() > 0 && result.len() >= limit as usize { // we have a 'virtual' entry as long as we have any new
+        count += 1;
     }
 
     rpcenv["total"] = Value::from(count);

src/api2/node/time.rs

@@ -1,4 +1,3 @@
-use chrono::prelude::*;
 use anyhow::{bail, format_err, Error};
 use serde_json::{json, Value};
 
@@ -57,10 +56,11 @@ fn read_etc_localtime() -> Result<String, Error> {
 )]
 /// Read server time and time zone settings.
 fn get_time(_param: Value) -> Result<Value, Error> {
-    let datetime = Local::now();
-    let offset = datetime.offset();
-    let time = datetime.timestamp();
-    let localtime = time + (offset.fix().local_minus_utc() as i64);
+    let time = proxmox::tools::time::epoch_i64();
+    let tm = proxmox::tools::time::localtime(time)?;
+    let offset = tm.tm_gmtoff;
+
+    let localtime = time + offset;
 
     Ok(json!({
         "timezone": read_etc_localtime()?,

src/api2/pull.rs

@@ -55,12 +55,13 @@ pub async fn get_pull_parameters(
         .password(Some(remote.password.clone()))
         .fingerprint(remote.fingerprint.clone());
 
-    let client = HttpClient::new(&remote.host, &remote.userid, options)?;
+    let src_repo = BackupRepository::new(Some(remote.userid.clone()), Some(remote.host.clone()), remote.port, remote_store.to_string());
+
+    let client = HttpClient::new(&src_repo.host(), src_repo.port(), &src_repo.user(), options)?;
     let _auth_info = client.login() // make sure we can auth
         .await
         .map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?;
 
-    let src_repo = BackupRepository::new(Some(remote.userid), Some(remote.host), remote_store.to_string());
 
     Ok((client, src_repo, tgt_store))
 }
@@ -176,7 +177,13 @@ async fn pull (
 
         worker.log(format!("sync datastore '{}' start", store));
 
-        pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, userid).await?;
+        let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, userid);
+        let future = select!{
+            success = pull_future.fuse() => success,
+            abort = worker.abort_future().map(|_| Err(format_err!("pull aborted"))) => abort,
+        };
+
+        let _ = future?;
 
         worker.log(format!("sync datastore '{}' end", store));
 

src/api2/reader.rs

@@ -1,4 +1,3 @@
-//use chrono::{Local, TimeZone};
 use anyhow::{bail, format_err, Error};
 use futures::*;
 use hyper::header::{self, HeaderValue, UPGRADE};
@@ -83,12 +82,12 @@ fn upgrade_to_backup_reader_protocol(
 
         let env_type = rpcenv.env_type();
 
-        let backup_dir = BackupDir::new(backup_type, backup_id, backup_time);
+        let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
         let path = datastore.base_path();
 
         //let files = BackupInfo::list_files(&path, &backup_dir)?;
 
-        let worker_id = format!("{}_{}_{}_{:08X}", store, backup_type, backup_id, backup_dir.backup_time().timestamp());
+        let worker_id = format!("{}_{}_{}_{:08X}", store, backup_type, backup_id, backup_dir.backup_time());
 
         WorkerTask::spawn("reader", Some(worker_id), userid.clone(), true, move |worker| {
             let mut env = ReaderEnvironment::new(
@@ -121,6 +120,7 @@ fn upgrade_to_backup_reader_protocol(
                         let window_size = 32*1024*1024; // max = (1 << 31) - 2
                         http.http2_initial_stream_window_size(window_size);
                         http.http2_initial_connection_window_size(window_size);
+                        http.http2_max_frame_size(4*1024*1024);
 
                         http.serve_connection(conn, service)
                             .map_err(Error::from)
@@ -229,8 +229,7 @@ fn download_chunk(
 
         env.debug(format!("download chunk {:?}", path));
 
-        let data = tokio::fs::read(path)
-            .await
+        let data = tools::runtime::block_in_place(|| std::fs::read(path))
             .map_err(move |err| http_err!(BAD_REQUEST, "reading file {:?} failed: {}", path2, err))?;
 
         let body = Body::from(data);
@@ -287,7 +286,7 @@ fn download_chunk_old(
 
 pub const API_METHOD_SPEEDTEST: ApiMethod = ApiMethod::new(
     &ApiHandler::AsyncHttp(&speedtest),
-    &ObjectSchema::new("Test 4M block download speed.", &[])
+    &ObjectSchema::new("Test 1M block download speed.", &[])
 );
 
 fn speedtest(

src/api2/status.rs

@@ -23,7 +23,6 @@ use crate::api2::types::{
 use crate::server;
 use crate::backup::{DataStore};
 use crate::config::datastore;
-use crate::tools::epoch_now_f64;
 use crate::tools::statistics::{linear_regression};
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::acl::{
@@ -74,6 +73,9 @@ use crate::config::acl::{
             },
         },
     },
+    access: {
+        permission: &Permission::Anybody,
+    },
 )]
 /// List Datastore usages and estimates
 fn datastore_status(
@@ -107,7 +109,7 @@ fn datastore_status(
         });
 
         let rrd_dir = format!("datastore/{}", store);
-        let now = epoch_now_f64()?;
+        let now = proxmox::tools::time::epoch_f64();
         let rrd_resolution = RRDTimeFrameResolution::Month;
         let rrd_mode = RRDMode::Average;
 
@@ -180,7 +182,7 @@ fn datastore_status(
     input: {
         properties: {
             since: {
-                type: u64,
+                type: i64,
                 description: "Only list tasks since this UNIX epoch.",
                 optional: true,
             },
@@ -198,6 +200,7 @@ fn datastore_status(
 )]
 /// List tasks.
 pub fn list_tasks(
+    since: Option<i64>,
     _param: Value,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<TaskListItem>, Error> {
@@ -207,13 +210,28 @@ pub fn list_tasks(
     let user_privs = user_info.lookup_privs(&userid, &["system", "tasks"]);
 
     let list_all = (user_privs & PRIV_SYS_AUDIT) != 0;
+    let since = since.unwrap_or_else(|| 0);
 
-    // TODO: replace with call that gets all task since 'since' epoch
-    let list: Vec<TaskListItem> = server::read_task_list()?
-        .into_iter()
-        .map(TaskListItem::from)
-        .filter(|entry| list_all || entry.user == userid)
-        .collect();
+    let list: Vec<TaskListItem> = server::TaskListInfoIterator::new(false)?
+        .take_while(|info| {
+            match info {
+                Ok(info) => info.upid.starttime > since,
+                Err(_) => false
+            }
+        })
+        .filter_map(|info| {
+            match info {
+                Ok(info) => {
+                    if list_all || info.upid.userid == userid {
+                        Some(Ok(TaskListItem::from(info)))
+                    } else {
+                        None
+                    }
+                }
+                Err(err) => Some(Err(err))
+            }
+        })
+        .collect::<Result<Vec<TaskListItem>, Error>>()?;
 
     Ok(list.into())
 }

src/api2/types/mod.rs

@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, schema::*};
 use proxmox::const_regex;
-use proxmox::{IPRE, IPV4RE, IPV6RE, IPV4OCTET, IPV6H16, IPV6LS32};
+use proxmox::{IPRE, IPRE_BRACKET, IPV4RE, IPV6RE, IPV4OCTET, IPV6H16, IPV6LS32};
 
 use crate::backup::CryptMode;
 use crate::server::UPID;
@@ -30,7 +30,7 @@ pub const FILENAME_FORMAT: ApiStringFormat = ApiStringFormat::VerifyFn(|name| {
 });
 
 macro_rules! DNS_LABEL { () => (r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9])?)") }
-macro_rules! DNS_NAME { () => (concat!(r"(?:", DNS_LABEL!() , r"\.)*", DNS_LABEL!())) }
+macro_rules! DNS_NAME { () => (concat!(r"(?:(?:", DNS_LABEL!() , r"\.)*", DNS_LABEL!(), ")")) }
 
 macro_rules! CIDR_V4_REGEX_STR { () => (concat!(r"(?:", IPV4RE!(), r"/\d{1,2})$")) }
 macro_rules! CIDR_V6_REGEX_STR { () => (concat!(r"(?:", IPV6RE!(), r"/\d{1,3})$")) }
@@ -63,9 +63,9 @@ const_regex!{
 
     pub DNS_NAME_REGEX =  concat!(r"^", DNS_NAME!(), r"$");
 
-    pub DNS_NAME_OR_IP_REGEX = concat!(r"^", DNS_NAME!(), "|",  IPRE!(), r"$");
+    pub DNS_NAME_OR_IP_REGEX = concat!(r"^(?:", DNS_NAME!(), "|",  IPRE!(), r")$");
 
-    pub BACKUP_REPO_URL_REGEX = concat!(r"^^(?:(?:(", USER_ID_REGEX_STR!(), ")@)?(", DNS_NAME!(), "|",  IPRE!() ,"):)?(", PROXMOX_SAFE_ID_REGEX_STR!(), r")$");
+    pub BACKUP_REPO_URL_REGEX = concat!(r"^^(?:(?:(", USER_ID_REGEX_STR!(), ")@)?(", DNS_NAME!(), "|",  IPRE_BRACKET!() ,"):)?(?:([0-9]{1,5}):)?(", PROXMOX_SAFE_ID_REGEX_STR!(), r")$");
 
     pub CERT_FINGERPRINT_SHA256_REGEX = r"^(?:[0-9a-fA-F][0-9a-fA-F])(?::[0-9a-fA-F][0-9a-fA-F]){31}$";
 
@@ -302,6 +302,11 @@ pub const PRUNE_SCHEDULE_SCHEMA: Schema = StringSchema::new(
     .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
     .schema();
 
+pub const VERIFY_SCHEDULE_SCHEMA: Schema = StringSchema::new(
+    "Run verify job at specified schedule.")
+    .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
+    .schema();
+
 pub const REMOTE_ID_SCHEMA: Schema = StringSchema::new("Remote ID.")
     .format(&PROXMOX_SAFE_ID_FORMAT)
     .min_length(3)
@@ -380,13 +385,24 @@ pub struct GroupListItem {
     pub owner: Option<Userid>,
 }
 
+#[api()]
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// Result of a verify operation.
+pub enum VerifyState {
+    /// Verification was successful
+    Ok,
+    /// Verification reported one or more errors
+    Failed,
+}
+
 #[api(
     properties: {
         upid: {
             schema: UPID_SCHEMA
         },
         state: {
-            type: String
+            type: VerifyState
         },
     },
 )]
@@ -395,8 +411,8 @@ pub struct GroupListItem {
 pub struct SnapshotVerifyState {
     /// UPID of the verify task
     pub upid: UPID,
-    /// State of the verification. "failed" or "ok"
-    pub state: String,
+    /// State of the verification. Enum.
+    pub state: VerifyState,
 }
 
 #[api(
@@ -559,6 +575,8 @@ pub struct GarbageCollectionStatus {
     pub pending_bytes: u64,
     /// Number of pending chunks (pending removal - kept for safety).
     pub pending_chunks: usize,
+    /// Number of chunks marked as .bad by verify that have been removed by GC.
+    pub removed_bad: usize,
 }
 
 impl Default for GarbageCollectionStatus {
@@ -573,6 +591,7 @@ impl Default for GarbageCollectionStatus {
             removed_chunks: 0,
             pending_bytes: 0,
             pending_chunks: 0,
+            removed_bad: 0,
         }
     }
 }
@@ -685,7 +704,7 @@ pub enum LinuxBondMode {
     /// Broadcast policy
     broadcast = 3,
     /// IEEE 802.3ad Dynamic link aggregation
-    //#[serde(rename = "802.3ad")]
+    #[serde(rename = "802.3ad")]
     ieee802_3ad = 4,
     /// Adaptive transmit load balancing
     balance_tlb = 5,
@@ -693,6 +712,23 @@ pub enum LinuxBondMode {
     balance_alb = 6,
 }
 
+#[api()]
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+#[allow(non_camel_case_types)]
+#[repr(u8)]
+/// Bond Transmit Hash Policy for LACP (802.3ad)
+pub enum BondXmitHashPolicy {
+    /// Layer 2
+    layer2 = 0,
+    /// Layer 2+3
+    #[serde(rename = "layer2+3")]
+    layer2_3 = 1,
+    /// Layer 3+4
+    #[serde(rename = "layer3+4")]
+    layer3_4 = 2,
+}
+
 #[api()]
 #[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "lowercase")]
@@ -798,7 +834,15 @@ pub const NETWORK_INTERFACE_LIST_SCHEMA: Schema = StringSchema::new(
         bond_mode: {
             type: LinuxBondMode,
             optional: true,
-        }
+        },
+        "bond-primary": {
+            schema: NETWORK_INTERFACE_NAME_SCHEMA,
+            optional: true,
+        },
+        bond_xmit_hash_policy: {
+            type: BondXmitHashPolicy,
+            optional: true,
+        },
     }
 )]
 #[derive(Debug, Serialize, Deserialize)]
@@ -855,6 +899,10 @@ pub struct Interface {
     pub slaves: Option<Vec<String>>,
     #[serde(skip_serializing_if="Option::is_none")]
     pub bond_mode: Option<LinuxBondMode>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    #[serde(rename = "bond-primary")]
+    pub bond_primary: Option<String>,
+    pub bond_xmit_hash_policy: Option<BondXmitHashPolicy>,
 }
 
 // Regression tests

src/auth_helpers.rs

@@ -11,7 +11,6 @@ use proxmox::tools::fs::{file_get_contents, replace_file, CreateOptions};
 use proxmox::try_block;
 
 use crate::api2::types::Userid;
-use crate::tools::epoch_now_u64;
 
 fn compute_csrf_secret_digest(
     timestamp: i64,
@@ -32,7 +31,7 @@ pub fn assemble_csrf_prevention_token(
     userid: &Userid,
 ) -> String {
 
-    let epoch = epoch_now_u64().unwrap() as i64;
+    let epoch = proxmox::tools::time::epoch_i64();
 
     let digest = compute_csrf_secret_digest(epoch, secret, userid);
 
@@ -69,7 +68,7 @@ pub fn verify_csrf_prevention_token(
             bail!("invalid signature.");
         }
 
-        let now = epoch_now_u64()? as i64;
+        let now = proxmox::tools::time::epoch_i64();
 
         let age = now - ttime;
         if age < min_age {

src/backup/backup_info.rs

@@ -4,8 +4,6 @@ use anyhow::{bail, format_err, Error};
 use regex::Regex;
 use std::os::unix::io::RawFd;
 
-use chrono::{DateTime, TimeZone, SecondsFormat, Utc};
-
 use std::path::{PathBuf, Path};
 use lazy_static::lazy_static;
 
@@ -105,8 +103,7 @@ impl BackupGroup {
         tools::scandir(libc::AT_FDCWD, &path, &BACKUP_DATE_REGEX, |l2_fd, backup_time, file_type| {
             if file_type != nix::dir::Type::Directory { return Ok(()); }
 
-            let dt = backup_time.parse::<DateTime<Utc>>()?;
-            let backup_dir = BackupDir::new(self.backup_type.clone(), self.backup_id.clone(), dt.timestamp());
+            let backup_dir = BackupDir::with_rfc3339(&self.backup_type, &self.backup_id, backup_time)?;
             let files = list_backup_files(l2_fd, backup_time)?;
 
             list.push(BackupInfo { backup_dir, files });
@@ -116,7 +113,7 @@ impl BackupGroup {
         Ok(list)
     }
 
-    pub fn last_successful_backup(&self,  base_path: &Path) -> Result<Option<DateTime<Utc>>, Error> {
+    pub fn last_successful_backup(&self,  base_path: &Path) -> Result<Option<i64>, Error> {
 
         let mut last = None;
 
@@ -142,11 +139,11 @@ impl BackupGroup {
                 }
             }
 
-            let dt = backup_time.parse::<DateTime<Utc>>()?;
-            if let Some(last_dt) = last {
-                if dt > last_dt { last = Some(dt); }
+            let timestamp = proxmox::tools::time::parse_rfc3339(backup_time)?;
+            if let Some(last_timestamp) = last {
+                if timestamp > last_timestamp { last = Some(timestamp); }
             } else {
-                last = Some(dt);
+                last = Some(timestamp);
             }
 
             Ok(())
@@ -203,45 +200,63 @@ pub struct BackupDir {
     /// Backup group
     group: BackupGroup,
     /// Backup timestamp
-    backup_time: DateTime<Utc>,
+    backup_time: i64,
+    // backup_time as rfc3339
+    backup_time_string: String
 }
 
 impl BackupDir {
 
-    pub fn new<T, U>(backup_type: T, backup_id: U, timestamp: i64) -> Self
+    pub fn new<T, U>(backup_type: T, backup_id: U, backup_time: i64) -> Result<Self, Error>
     where
         T: Into<String>,
         U: Into<String>,
     {
-        // Note: makes sure that nanoseconds is 0
-        Self {
-            group: BackupGroup::new(backup_type.into(), backup_id.into()),
-            backup_time: Utc.timestamp(timestamp, 0),
-        }
+        let group = BackupGroup::new(backup_type.into(), backup_id.into());
+        BackupDir::with_group(group, backup_time)
     }
-    pub fn new_with_group(group: BackupGroup, timestamp: i64) -> Self {
-        Self { group, backup_time: Utc.timestamp(timestamp, 0) }
+
+    pub fn with_rfc3339<T,U,V>(backup_type: T, backup_id: U, backup_time_string: V) -> Result<Self, Error>
+    where
+        T: Into<String>,
+        U: Into<String>,
+        V: Into<String>,
+    {
+        let backup_time_string =  backup_time_string.into();
+        let backup_time = proxmox::tools::time::parse_rfc3339(&backup_time_string)?;
+        let group = BackupGroup::new(backup_type.into(), backup_id.into());
+        Ok(Self { group, backup_time, backup_time_string })
+    }
+
+    pub fn with_group(group: BackupGroup, backup_time: i64) -> Result<Self, Error> {
+        let backup_time_string = Self::backup_time_to_string(backup_time)?;
+        Ok(Self { group, backup_time, backup_time_string })
     }
 
     pub fn group(&self) -> &BackupGroup {
         &self.group
     }
 
-    pub fn backup_time(&self) -> DateTime<Utc> {
+    pub fn backup_time(&self) -> i64 {
         self.backup_time
     }
 
+    pub fn backup_time_string(&self) -> &str {
+        &self.backup_time_string
+    }
+
     pub fn relative_path(&self) ->  PathBuf  {
 
         let mut relative_path = self.group.group_path();
 
-        relative_path.push(Self::backup_time_to_string(self.backup_time));
+        relative_path.push(self.backup_time_string.clone());
 
         relative_path
     }
 
-    pub fn backup_time_to_string(backup_time: DateTime<Utc>) -> String {
-        backup_time.to_rfc3339_opts(SecondsFormat::Secs, true)
+    pub fn backup_time_to_string(backup_time: i64) -> Result<String, Error> {
+        // fixme: can this fail? (avoid unwrap)
+        proxmox::tools::time::epoch_to_rfc3339_utc(backup_time)
     }
 }
 
@@ -255,9 +270,11 @@ impl std::str::FromStr for BackupDir {
         let cap = SNAPSHOT_PATH_REGEX.captures(path)
             .ok_or_else(|| format_err!("unable to parse backup snapshot path '{}'", path))?;
 
-        let group = BackupGroup::new(cap.get(1).unwrap().as_str(), cap.get(2).unwrap().as_str());
-        let backup_time = cap.get(3).unwrap().as_str().parse::<DateTime<Utc>>()?;
-        Ok(BackupDir::from((group, backup_time.timestamp())))
+        BackupDir::with_rfc3339(
+            cap.get(1).unwrap().as_str(),
+            cap.get(2).unwrap().as_str(),
+            cap.get(3).unwrap().as_str(),
+        )
     }
 }
 
@@ -265,14 +282,7 @@ impl std::fmt::Display for BackupDir {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let backup_type = self.group.backup_type();
         let id = self.group.backup_id();
-        let time = Self::backup_time_to_string(self.backup_time);
-        write!(f, "{}/{}/{}", backup_type, id, time)
-    }
-}
-
-impl From<(BackupGroup, i64)> for BackupDir {
-    fn from((group, timestamp): (BackupGroup, i64)) -> Self {
-        Self { group, backup_time: Utc.timestamp(timestamp, 0) }
+        write!(f, "{}/{}/{}", backup_type, id, self.backup_time_string)
     }
 }
 
@@ -330,13 +340,12 @@ impl BackupInfo {
             if file_type != nix::dir::Type::Directory { return Ok(()); }
             tools::scandir(l0_fd, backup_type, &BACKUP_ID_REGEX, |l1_fd, backup_id, file_type| {
                 if file_type != nix::dir::Type::Directory { return Ok(()); }
-                tools::scandir(l1_fd, backup_id, &BACKUP_DATE_REGEX, |l2_fd, backup_time, file_type| {
+                tools::scandir(l1_fd, backup_id, &BACKUP_DATE_REGEX, |l2_fd, backup_time_string, file_type| {
                     if file_type != nix::dir::Type::Directory { return Ok(()); }
 
-                    let dt = backup_time.parse::<DateTime<Utc>>()?;
-                    let backup_dir = BackupDir::new(backup_type, backup_id, dt.timestamp());
+                    let backup_dir = BackupDir::with_rfc3339(backup_type, backup_id, backup_time_string)?;
 
-                    let files = list_backup_files(l2_fd, backup_time)?;
+                    let files = list_backup_files(l2_fd, backup_time_string)?;
 
                     list.push(BackupInfo { backup_dir, files });
 

src/backup/catalog.rs

@@ -5,7 +5,6 @@ use std::io::{Read, Write, Seek, SeekFrom};
 use std::os::unix::ffi::OsStrExt;
 
 use anyhow::{bail, format_err, Error};
-use chrono::offset::{TimeZone, Local};
 
 use pathpatterns::{MatchList, MatchType};
 use proxmox::tools::io::ReadExt;
@@ -533,17 +532,17 @@ impl <R: Read + Seek> CatalogReader<R> {
                     self.dump_dir(&path, pos)?;
                 }
                 CatalogEntryType::File => {
-                    let dt = Local
-                        .timestamp_opt(mtime as i64, 0)
-                        .single() // chrono docs say timestamp_opt can only be None or Single!
-                        .unwrap_or_else(|| Local.timestamp(0, 0));
+                    let mut mtime_string = mtime.to_string();
+                    if let Ok(s) = proxmox::tools::time::strftime_local("%FT%TZ", mtime as i64) {
+                        mtime_string = s;
+                    }
 
                     println!(
                         "{} {:?} {} {}",
                         etype,
                         path,
                         size,
-                        dt.to_rfc3339_opts(chrono::SecondsFormat::Secs, false),
+                        mtime_string,
                     );
                 }
                 _ => {

src/backup/chunk_store.rs

@@ -104,12 +104,11 @@ impl ChunkStore {
             }
             let percentage = (i*100)/(64*1024);
             if percentage != last_percentage {
-                eprintln!("{}%", percentage);
+                // eprintln!("ChunkStore::create {}%", percentage);
                 last_percentage = percentage;
             }
         }
 
-
         Self::open(name, base)
     }
 
@@ -187,7 +186,7 @@ impl ChunkStore {
     pub fn get_chunk_iterator(
         &self,
     ) -> Result<
-        impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize)> + std::iter::FusedIterator,
+        impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize, bool)> + std::iter::FusedIterator,
         Error
     > {
         use nix::dir::Dir;
@@ -219,19 +218,21 @@ impl ChunkStore {
                         Some(Ok(entry)) => {
                             // skip files if they're not a hash
                             let bytes = entry.file_name().to_bytes();
-                            if bytes.len() != 64 {
+                            if bytes.len() != 64 && bytes.len() != 64 + ".0.bad".len() {
                                 continue;
                             }
-                            if !bytes.iter().all(u8::is_ascii_hexdigit) {
+                            if !bytes.iter().take(64).all(u8::is_ascii_hexdigit) {
                                 continue;
                             }
-                            return Some((Ok(entry), percentage));
+
+                            let bad = bytes.ends_with(".bad".as_bytes());
+                            return Some((Ok(entry), percentage, bad));
                         }
                         Some(Err(err)) => {
                             // stop after first error
                             done = true;
                             // and pass the error through:
-                            return Some((Err(err), percentage));
+                            return Some((Err(err), percentage, false));
                         }
                         None => (), // open next directory
                     }
@@ -261,7 +262,7 @@ impl ChunkStore {
                         // other errors are fatal, so end our iteration
                         done = true;
                         // and pass the error through:
-                        return Some((Err(format_err!("unable to read subdir '{}' - {}", subdir, err)), percentage));
+                        return Some((Err(format_err!("unable to read subdir '{}' - {}", subdir, err)), percentage, false));
                     }
                 }
             }
@@ -280,6 +281,7 @@ impl ChunkStore {
         worker: &WorkerTask,
     ) -> Result<(), Error> {
         use nix::sys::stat::fstatat;
+        use nix::unistd::{unlinkat, UnlinkatFlags};
 
         let mut min_atime = phase1_start_time - 3600*24; // at least 24h (see mount option relatime)
 
@@ -292,10 +294,10 @@ impl ChunkStore {
         let mut last_percentage = 0;
         let mut chunk_count = 0;
 
-        for (entry, percentage) in self.get_chunk_iterator()? {
+        for (entry, percentage, bad) in self.get_chunk_iterator()? {
             if last_percentage != percentage {
                 last_percentage = percentage;
-                worker.log(format!("{}%, processed {} chunks", percentage, chunk_count));
+                worker.log(format!("percentage done: phase2 {}% (processed {} chunks)", percentage, chunk_count));
             }
 
             worker.fail_on_abort()?;
@@ -321,14 +323,47 @@ impl ChunkStore {
             let lock = self.mutex.lock();
 
             if let Ok(stat) = fstatat(dirfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
-                if stat.st_atime < min_atime {
+                if bad {
+                    // filename validity checked in iterator
+                    let orig_filename = std::ffi::CString::new(&filename.to_bytes()[..64])?;
+                    match fstatat(
+                        dirfd,
+                        orig_filename.as_c_str(),
+                        nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW)
+                    {
+                        Ok(_) => {
+                            match unlinkat(Some(dirfd), filename, UnlinkatFlags::NoRemoveDir) {
+                                Err(err) =>
+                                    worker.warn(format!(
+                                        "unlinking corrupt chunk {:?} failed on store '{}' - {}",
+                                        filename,
+                                        self.name,
+                                        err,
+                                    )),
+                                Ok(_) => {
+                                    status.removed_bad += 1;
+                                    status.removed_bytes += stat.st_size as u64;
+                                }
+                            }
+                        },
+                        Err(nix::Error::Sys(nix::errno::Errno::ENOENT)) => {
+                            // chunk hasn't been rewritten yet, keep .bad file
+                        },
+                        Err(err) => {
+                            // some other error, warn user and keep .bad file around too
+                            worker.warn(format!(
+                                "error during stat on '{:?}' - {}",
+                                orig_filename,
+                                err,
+                            ));
+                        }
+                    }
+                } else if stat.st_atime < min_atime {
                     //let age = now - stat.st_atime;
                     //println!("UNLINK {}  {:?}", age/(3600*24), filename);
-                    let res = unsafe { libc::unlinkat(dirfd, filename.as_ptr(), 0) };
-                    if res != 0 {
-                        let err = nix::Error::last();
+                    if let Err(err) = unlinkat(Some(dirfd), filename, UnlinkatFlags::NoRemoveDir) {
                         bail!(
-                            "unlink chunk {:?} failed on store '{}' - {}",
+                            "unlinking chunk {:?} failed on store '{}' - {}",
                             filename,
                             self.name,
                             err,
@@ -366,6 +401,7 @@ impl ChunkStore {
 
         if let Ok(metadata) = std::fs::metadata(&chunk_path) {
             if metadata.is_file() {
+                self.touch_chunk(digest)?;
                 return Ok((true, metadata.len()));
             } else {
                 bail!("Got unexpected file type on store '{}' for chunk {}", self.name, digest_str);

src/backup/crypt_config.rs

@@ -10,7 +10,6 @@
 use std::io::Write;
 
 use anyhow::{bail, Error};
-use chrono::{Local, TimeZone, DateTime};
 use openssl::hash::MessageDigest;
 use openssl::pkcs5::pbkdf2_hmac;
 use openssl::symm::{decrypt_aead, Cipher, Crypter, Mode};
@@ -216,10 +215,10 @@ impl CryptConfig {
     pub fn generate_rsa_encoded_key(
         &self,
         rsa: openssl::rsa::Rsa<openssl::pkey::Public>,
-        created: DateTime<Local>,
+        created: i64,
     ) -> Result<Vec<u8>, Error> {
 
-        let modified = Local.timestamp(Local::now().timestamp(), 0);
+        let modified = proxmox::tools::time::epoch_i64();
         let key_config = super::KeyConfig { kdf: None, created, modified, data: self.enc_key.to_vec() };
         let data = serde_json::to_string(&key_config)?.as_bytes().to_vec();
 

src/backup/data_blob.rs

@@ -72,7 +72,7 @@ impl DataBlob {
     }
 
     // verify the CRC32 checksum
-    fn verify_crc(&self) -> Result<(), Error> {
+    pub fn verify_crc(&self) -> Result<(), Error> {
         let expected_crc = self.compute_crc();
         if expected_crc != self.crc() {
             bail!("Data blob has wrong CRC checksum.");
@@ -198,7 +198,10 @@ impl DataBlob {
             Ok(data)
         } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
             let data_start = std::mem::size_of::<DataBlobHeader>();
-            let data = zstd::block::decompress(&self.raw_data[data_start..], MAX_BLOB_SIZE)?;
+            let mut reader = &self.raw_data[data_start..];
+            let data = zstd::stream::decode_all(&mut reader)?;
+            // zstd::block::decompress is abou 10% slower
+            // let data = zstd::block::decompress(&self.raw_data[data_start..], MAX_BLOB_SIZE)?;
             if let Some(digest) = digest {
                 Self::verify_digest(&data, None, digest)?;
             }
@@ -268,6 +271,12 @@ impl DataBlob {
         }
     }
 
+    /// Returns if chunk is encrypted
+    pub fn is_encrypted(&self) -> bool {
+        let magic = self.magic();
+        magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0
+    }
+
     /// Verify digest and data length for unencrypted chunks.
     ///
     /// To do that, we need to decompress data first. Please note that
@@ -304,7 +313,7 @@ impl DataBlob {
 
         let digest = match config {
             Some(config) => config.compute_digest(data),
-            None => openssl::sha::sha256(&data),
+            None => openssl::sha::sha256(data),
         };
         if &digest != expected_digest {
             bail!("detected chunk with wrong digest.");

src/backup/datastore.rs

@@ -6,7 +6,6 @@ use std::convert::TryFrom;
 
 use anyhow::{bail, format_err, Error};
 use lazy_static::lazy_static;
-use chrono::{DateTime, Utc};
 use serde_json::Value;
 
 use proxmox::tools::fs::{replace_file, CreateOptions};
@@ -71,6 +70,10 @@ impl DataStore {
 
         let path = store_config["path"].as_str().unwrap();
 
+        Self::open_with_path(store_name, Path::new(path))
+    }
+
+    pub fn open_with_path(store_name: &str, path: &Path) -> Result<Self, Error> {
         let chunk_store = ChunkStore::open(store_name, path)?;
 
         let gc_status = GarbageCollectionStatus::default();
@@ -85,7 +88,7 @@ impl DataStore {
     pub fn get_chunk_iterator(
         &self,
     ) -> Result<
-        impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize)>,
+        impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize, bool)>,
         Error
     > {
         self.chunk_store.get_chunk_iterator()
@@ -242,7 +245,7 @@ impl DataStore {
     /// Returns the time of the last successful backup
     ///
     /// Or None if there is no backup in the group (or the group dir does not exist).
-    pub fn last_successful_backup(&self, backup_group: &BackupGroup) -> Result<Option<DateTime<Utc>>, Error> {
+    pub fn last_successful_backup(&self, backup_group: &BackupGroup) -> Result<Option<i64>, Error> {
         let base_path = self.base_path();
         let mut group_path = base_path.clone();
         group_path.push(backup_group.group_path());
@@ -430,6 +433,12 @@ impl DataStore {
 
         let image_list = self.list_images()?;
 
+        let image_count = image_list.len();
+
+        let mut done = 0;
+
+        let mut last_percentage: usize = 0;
+
         for path in image_list {
 
             worker.fail_on_abort()?;
@@ -444,6 +453,14 @@ impl DataStore {
                     self.index_mark_used_chunks(index, &path, status, worker)?;
                 }
             }
+            done += 1;
+
+            let percentage = done*100/image_count;
+            if percentage > last_percentage {
+                worker.log(format!("percentage done: phase1 {}% ({} of {} index files)",
+                                   percentage, done, image_count));
+                last_percentage = percentage;
+            }
         }
 
         Ok(())
@@ -461,9 +478,12 @@ impl DataStore {
 
         if let Ok(ref mut _mutex) = self.gc_mutex.try_lock() {
 
+            // avoids that we run GC if an old daemon process has still a
+            // running backup writer, which is not save as we have no "oldest
+            // writer" information and thus no safe atime cutoff
             let _exclusive_lock =  self.chunk_store.try_exclusive_lock()?;
 
-            let phase1_start_time = unsafe { libc::time(std::ptr::null_mut()) };
+            let phase1_start_time = proxmox::tools::time::epoch_i64();
             let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(phase1_start_time);
 
             let mut gc_status = GarbageCollectionStatus::default();
@@ -481,6 +501,9 @@ impl DataStore {
             if gc_status.pending_bytes > 0 {
                 worker.log(&format!("Pending removals: {} (in {} chunks)", HumanByte::from(gc_status.pending_bytes), gc_status.pending_chunks));
             }
+            if gc_status.removed_bad > 0 {
+                worker.log(&format!("Removed bad files: {}", gc_status.removed_bad));
+            }
 
             worker.log(&format!("Original data usage: {}", HumanByte::from(gc_status.index_data_bytes)));
 

src/backup/dynamic_index.rs

@@ -21,14 +21,14 @@ use super::read_chunk::ReadChunk;
 use super::Chunker;
 use super::IndexFile;
 use super::{DataBlob, DataChunkBuilder};
-use crate::tools::{self, epoch_now_u64};
+use crate::tools;
 
 /// Header format definition for dynamic index files (`.dixd`)
 #[repr(C)]
 pub struct DynamicIndexHeader {
     pub magic: [u8; 8],
     pub uuid: [u8; 16],
-    pub ctime: u64,
+    pub ctime: i64,
     /// Sha256 over the index ``SHA256(offset1||digest1||offset2||digest2||...)``
     pub index_csum: [u8; 32],
     reserved: [u8; 4032], // overall size is one page (4096 bytes)
@@ -77,7 +77,7 @@ pub struct DynamicIndexReader {
     pub size: usize,
     index: Mmap<DynamicEntry>,
     pub uuid: [u8; 16],
-    pub ctime: u64,
+    pub ctime: i64,
     pub index_csum: [u8; 32],
 }
 
@@ -107,7 +107,7 @@ impl DynamicIndexReader {
             bail!("got unknown magic number");
         }
 
-        let ctime = u64::from_le(header.ctime);
+        let ctime = proxmox::tools::time::epoch_i64();
 
         let rawfd = file.as_raw_fd();
 
@@ -480,7 +480,7 @@ pub struct DynamicIndexWriter {
     tmp_filename: PathBuf,
     csum: Option<openssl::sha::Sha256>,
     pub uuid: [u8; 16],
-    pub ctime: u64,
+    pub ctime: i64,
 }
 
 impl Drop for DynamicIndexWriter {
@@ -506,13 +506,13 @@ impl DynamicIndexWriter {
 
         let mut writer = BufWriter::with_capacity(1024 * 1024, file);
 
-        let ctime = epoch_now_u64()?;
+        let ctime = proxmox::tools::time::epoch_i64();
 
         let uuid = Uuid::generate();
 
         let mut header = DynamicIndexHeader::zeroed();
         header.magic = super::DYNAMIC_SIZED_CHUNK_INDEX_1_0;
-        header.ctime = u64::to_le(ctime);
+        header.ctime = i64::to_le(ctime);
         header.uuid = *uuid.as_bytes();
         // header.index_csum = [0u8; 32];
         writer.write_all(header.as_bytes())?;

src/backup/fixed_index.rs

@@ -4,9 +4,8 @@ use std::io::{Seek, SeekFrom};
 use super::chunk_stat::*;
 use super::chunk_store::*;
 use super::{IndexFile, ChunkReadInfo};
-use crate::tools::{self, epoch_now_u64};
+use crate::tools;
 
-use chrono::{Local, TimeZone};
 use std::fs::File;
 use std::io::Write;
 use std::os::unix::io::AsRawFd;
@@ -23,7 +22,7 @@ use proxmox::tools::Uuid;
 pub struct FixedIndexHeader {
     pub magic: [u8; 8],
     pub uuid: [u8; 16],
-    pub ctime: u64,
+    pub ctime: i64,
     /// Sha256 over the index ``SHA256(digest1||digest2||...)``
     pub index_csum: [u8; 32],
     pub size: u64,
@@ -41,7 +40,7 @@ pub struct FixedIndexReader {
     index_length: usize,
     index: *mut u8,
     pub uuid: [u8; 16],
-    pub ctime: u64,
+    pub ctime: i64,
     pub index_csum: [u8; 32],
 }
 
@@ -82,7 +81,7 @@ impl FixedIndexReader {
         }
 
         let size = u64::from_le(header.size);
-        let ctime = u64::from_le(header.ctime);
+        let ctime = i64::from_le(header.ctime);
         let chunk_size = u64::from_le(header.chunk_size);
 
         let index_length = ((size + chunk_size - 1) / chunk_size) as usize;
@@ -148,10 +147,13 @@ impl FixedIndexReader {
     pub fn print_info(&self) {
         println!("Size: {}", self.size);
         println!("ChunkSize: {}", self.chunk_size);
-        println!(
-            "CTime: {}",
-            Local.timestamp(self.ctime as i64, 0).format("%c")
-        );
+
+        let mut ctime_str = self.ctime.to_string();
+        if let Ok(s) = proxmox::tools::time::strftime_local("%c",self.ctime) {
+            ctime_str = s;
+        }
+
+        println!("CTime: {}", ctime_str);
         println!("UUID: {:?}", self.uuid);
     }
 }
@@ -228,7 +230,7 @@ pub struct FixedIndexWriter {
     index_length: usize,
     index: *mut u8,
     pub uuid: [u8; 16],
-    pub ctime: u64,
+    pub ctime: i64,
 }
 
 // `index` is mmap()ed which cannot be thread-local so should be sendable
@@ -271,7 +273,7 @@ impl FixedIndexWriter {
             panic!("got unexpected header size");
         }
 
-        let ctime = epoch_now_u64()?;
+        let ctime = proxmox::tools::time::epoch_i64();
 
         let uuid = Uuid::generate();
 
@@ -279,7 +281,7 @@ impl FixedIndexWriter {
         let header = unsafe { &mut *(buffer.as_ptr() as *mut FixedIndexHeader) };
 
         header.magic = super::FIXED_SIZED_CHUNK_INDEX_1_0;
-        header.ctime = u64::to_le(ctime);
+        header.ctime = i64::to_le(ctime);
         header.size = u64::to_le(size as u64);
         header.chunk_size = u64::to_le(chunk_size as u64);
         header.uuid = *uuid.as_bytes();

src/backup/key_derivation.rs

@@ -1,7 +1,6 @@
 use anyhow::{bail, format_err, Context, Error};
 
 use serde::{Deserialize, Serialize};
-use chrono::{Local, TimeZone, DateTime};
 
 use proxmox::tools::fs::{file_get_contents, replace_file, CreateOptions};
 use proxmox::try_block;
@@ -61,10 +60,10 @@ impl KeyDerivationConfig {
 #[derive(Deserialize, Serialize, Debug)]
 pub struct KeyConfig {
     pub kdf: Option<KeyDerivationConfig>,
-    #[serde(with = "proxmox::tools::serde::date_time_as_rfc3339")]
-    pub created: DateTime<Local>,
-    #[serde(with = "proxmox::tools::serde::date_time_as_rfc3339")]
-    pub modified: DateTime<Local>,
+    #[serde(with = "proxmox::tools::serde::epoch_as_rfc3339")]
+    pub created: i64,
+    #[serde(with = "proxmox::tools::serde::epoch_as_rfc3339")]
+    pub modified: i64,
     #[serde(with = "proxmox::tools::serde::bytes_as_base64")]
     pub data: Vec<u8>,
  }
@@ -136,7 +135,7 @@ pub fn encrypt_key_with_passphrase(
     enc_data.extend_from_slice(&tag);
     enc_data.extend_from_slice(&encrypted_key);
 
-    let created =  Local.timestamp(Local::now().timestamp(), 0);
+    let created = proxmox::tools::time::epoch_i64();
 
     Ok(KeyConfig {
         kdf: Some(kdf),
@@ -149,7 +148,7 @@ pub fn encrypt_key_with_passphrase(
 pub fn load_and_decrypt_key(
     path: &std::path::Path,
     passphrase: &dyn Fn() -> Result<Vec<u8>, Error>,
-) -> Result<([u8;32], DateTime<Local>), Error> {
+) -> Result<([u8;32], i64), Error> {
     do_load_and_decrypt_key(path, passphrase)
         .with_context(|| format!("failed to load decryption key from {:?}", path))
 }
@@ -157,14 +156,14 @@ pub fn load_and_decrypt_key(
 fn do_load_and_decrypt_key(
     path: &std::path::Path,
     passphrase: &dyn Fn() -> Result<Vec<u8>, Error>,
-) -> Result<([u8;32], DateTime<Local>), Error> {
+) -> Result<([u8;32], i64), Error> {
     decrypt_key(&file_get_contents(&path)?, passphrase)
 }
 
 pub fn decrypt_key(
     mut keydata: &[u8],
     passphrase: &dyn Fn() -> Result<Vec<u8>, Error>,
-) -> Result<([u8;32], DateTime<Local>), Error> {
+) -> Result<([u8;32], i64), Error> {
     let key_config: KeyConfig = serde_json::from_reader(&mut keydata)?;
 
     let raw_data = key_config.data;

src/backup/manifest.rs

@@ -103,7 +103,7 @@ impl BackupManifest {
         Self {
             backup_type: snapshot.group().backup_type().into(),
             backup_id: snapshot.group().backup_id().into(),
-            backup_time: snapshot.backup_time().timestamp(),
+            backup_time: snapshot.backup_time(),
             files: Vec::new(),
             unprotected: json!({}),
             signature: None,

src/backup/prune.rs

@@ -2,18 +2,16 @@ use anyhow::{Error};
 use std::collections::{HashMap, HashSet};
 use std::path::PathBuf;
 
-use chrono::{DateTime, Timelike, Datelike, Local};
-
-use super::{BackupDir, BackupInfo};
+use super::BackupInfo;
 
 enum PruneMark { Keep, KeepPartial, Remove }
 
-fn mark_selections<F: Fn(DateTime<Local>, &BackupInfo) -> String> (
+fn mark_selections<F: Fn(&BackupInfo) -> Result<String, Error>> (
     mark: &mut HashMap<PathBuf, PruneMark>,
     list: &Vec<BackupInfo>,
     keep: usize,
     select_id: F,
-) {
+) -> Result<(), Error> {
 
     let mut include_hash = HashSet::new();
 
@@ -21,8 +19,7 @@ fn mark_selections<F: Fn(DateTime<Local>, &BackupInfo) -> String> (
     for info in list {
         let backup_id = info.backup_dir.relative_path();
         if let Some(PruneMark::Keep) = mark.get(&backup_id) {
-            let local_time = info.backup_dir.backup_time().with_timezone(&Local);
-            let sel_id: String = select_id(local_time, &info);
+            let sel_id: String = select_id(&info)?;
             already_included.insert(sel_id);
         }
     }
@@ -30,8 +27,7 @@ fn mark_selections<F: Fn(DateTime<Local>, &BackupInfo) -> String> (
     for info in list {
         let backup_id = info.backup_dir.relative_path();
         if let Some(_) = mark.get(&backup_id) { continue; }
-        let local_time = info.backup_dir.backup_time().with_timezone(&Local);
-        let sel_id: String = select_id(local_time, &info);
+        let sel_id: String = select_id(&info)?;
 
         if already_included.contains(&sel_id) { continue; }
 
@@ -43,6 +39,8 @@ fn mark_selections<F: Fn(DateTime<Local>, &BackupInfo) -> String> (
             mark.insert(backup_id, PruneMark::Remove);
         }
     }
+
+    Ok(())
 }
 
 fn remove_incomplete_snapshots(
@@ -182,44 +180,43 @@ pub fn compute_prune_info(
     remove_incomplete_snapshots(&mut mark, &list);
 
     if let Some(keep_last) = options.keep_last {
-        mark_selections(&mut mark, &list, keep_last as usize, |_local_time, info| {
-            BackupDir::backup_time_to_string(info.backup_dir.backup_time())
-        });
+        mark_selections(&mut mark, &list, keep_last as usize, |info| {
+            Ok(info.backup_dir.backup_time_string().to_owned())
+        })?;
     }
 
+    use proxmox::tools::time::strftime_local;
+
     if let Some(keep_hourly) = options.keep_hourly {
-        mark_selections(&mut mark, &list, keep_hourly as usize, |local_time, _info| {
-            format!("{}/{}/{}/{}", local_time.year(), local_time.month(),
-                    local_time.day(), local_time.hour())
-        });
+        mark_selections(&mut mark, &list, keep_hourly as usize, |info| {
+            strftime_local("%Y/%m/%d/%H", info.backup_dir.backup_time())
+        })?;
     }
 
     if let Some(keep_daily) = options.keep_daily {
-        mark_selections(&mut mark, &list, keep_daily as usize, |local_time, _info| {
-            format!("{}/{}/{}", local_time.year(), local_time.month(), local_time.day())
-        });
+        mark_selections(&mut mark, &list, keep_daily as usize, |info| {
+            strftime_local("%Y/%m/%d", info.backup_dir.backup_time())
+        })?;
     }
 
     if let Some(keep_weekly) = options.keep_weekly {
-        mark_selections(&mut mark, &list, keep_weekly as usize, |local_time, _info| {
-            let iso_week = local_time.iso_week();
-            let week = iso_week.week();
-            // Note: This year number might not match the calendar year number.
-            let iso_week_year = iso_week.year();
-            format!("{}/{}", iso_week_year, week)
-        });
+        mark_selections(&mut mark, &list, keep_weekly as usize, |info| {
+            // Note: Use iso-week year/week here. This year number
+            // might not match the calendar year number.
+            strftime_local("%G/%V", info.backup_dir.backup_time())
+        })?;
     }
 
     if let Some(keep_monthly) = options.keep_monthly {
-        mark_selections(&mut mark, &list, keep_monthly as usize, |local_time, _info| {
-            format!("{}/{}", local_time.year(), local_time.month())
-        });
+        mark_selections(&mut mark, &list, keep_monthly as usize, |info| {
+            strftime_local("%Y/%m", info.backup_dir.backup_time())
+        })?;
     }
 
     if let Some(keep_yearly) = options.keep_yearly {
-        mark_selections(&mut mark, &list, keep_yearly as usize, |local_time, _info| {
-            format!("{}/{}", local_time.year(), local_time.year())
-        });
+        mark_selections(&mut mark, &list, keep_yearly as usize, |info| {
+            strftime_local("%Y", info.backup_dir.backup_time())
+        })?;
     }
 
     let prune_info: Vec<(BackupInfo, bool)> = list.into_iter()

src/backup/verify.rs

@@ -1,17 +1,29 @@
 use std::collections::HashSet;
+use std::sync::{Arc, Mutex};
+use std::sync::atomic::{Ordering, AtomicUsize};
+use std::time::Instant;
 
 use anyhow::{bail, format_err, Error};
 
-use crate::server::WorkerTask;
-use crate::api2::types::*;
-
-use super::{
-    DataStore, BackupGroup, BackupDir, BackupInfo, IndexFile,
-    CryptMode,
-    FileInfo, ArchiveType, archive_type,
+use crate::{
+    server::WorkerTask,
+    api2::types::*,
+    tools::ParallelHandler,
+    backup::{
+        DataStore,
+        DataBlob,
+        BackupGroup,
+        BackupDir,
+        BackupInfo,
+        IndexFile,
+        CryptMode,
+        FileInfo,
+        ArchiveType,
+        archive_type,
+    },
 };
 
-fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
+fn verify_blob(datastore: Arc<DataStore>, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
 
     let blob = datastore.load_blob(backup_dir, &info.filename)?;
 
@@ -36,74 +48,141 @@ fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -
     }
 }
 
+fn rename_corrupted_chunk(
+    datastore: Arc<DataStore>,
+    digest: &[u8;32],
+    worker: Arc<WorkerTask>,
+) {
+    let (path, digest_str) = datastore.chunk_path(digest);
+
+    let mut counter = 0;
+    let mut new_path = path.clone();
+    loop {
+        new_path.set_file_name(format!("{}.{}.bad", digest_str, counter));
+        if new_path.exists() && counter < 9 { counter += 1; } else { break; }
+    }
+
+    match std::fs::rename(&path, &new_path) {
+        Ok(_) => {
+            worker.log(format!("corrupted chunk renamed to {:?}", &new_path));
+        },
+        Err(err) => {
+            match err.kind() {
+                std::io::ErrorKind::NotFound => { /* ignored */ },
+                _ => worker.log(format!("could not rename corrupted chunk {:?} - {}", &path, err))
+            }
+        }
+    };
+}
+
 fn verify_index_chunks(
-    datastore: &DataStore,
-    index: Box<dyn IndexFile>,
-    verified_chunks: &mut HashSet<[u8;32]>,
-    corrupt_chunks: &mut HashSet<[u8; 32]>,
+    datastore: Arc<DataStore>,
+    index: Box<dyn IndexFile + Send>,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8; 32]>>>,
     crypt_mode: CryptMode,
-    worker: &WorkerTask,
+    worker: Arc<WorkerTask>,
 ) -> Result<(), Error> {
 
-    let mut errors = 0;
+    let errors = Arc::new(AtomicUsize::new(0));
+
+    let start_time = Instant::now();
+
+    let mut read_bytes = 0;
+    let mut decoded_bytes = 0;
+
+    let worker2 = Arc::clone(&worker);
+    let datastore2 = Arc::clone(&datastore);
+    let corrupt_chunks2 = Arc::clone(&corrupt_chunks);
+    let verified_chunks2 = Arc::clone(&verified_chunks);
+    let errors2 = Arc::clone(&errors);
+
+    let decoder_pool = ParallelHandler::new(
+        "verify chunk decoder", 4,
+        move |(chunk, digest, size): (DataBlob, [u8;32], u64)| {
+            let chunk_crypt_mode = match chunk.crypt_mode() {
+                Err(err) => {
+                    corrupt_chunks2.lock().unwrap().insert(digest);
+                    worker2.log(format!("can't verify chunk, unknown CryptMode - {}", err));
+                    errors2.fetch_add(1, Ordering::SeqCst);
+                    return Ok(());
+                },
+                Ok(mode) => mode,
+            };
+
+            if chunk_crypt_mode != crypt_mode {
+                worker2.log(format!(
+                    "chunk CryptMode {:?} does not match index CryptMode {:?}",
+                    chunk_crypt_mode,
+                    crypt_mode
+                ));
+                errors2.fetch_add(1, Ordering::SeqCst);
+            }
+
+            if let Err(err) = chunk.verify_unencrypted(size as usize, &digest) {
+                corrupt_chunks2.lock().unwrap().insert(digest);
+                worker2.log(format!("{}", err));
+                errors2.fetch_add(1, Ordering::SeqCst);
+                rename_corrupted_chunk(datastore2.clone(), &digest, worker2.clone());
+            } else {
+                verified_chunks2.lock().unwrap().insert(digest);
+            }
+
+            Ok(())
+        }
+    );
+
     for pos in 0..index.index_count() {
 
         worker.fail_on_abort()?;
+        crate::tools::fail_on_shutdown()?;
 
         let info = index.chunk_info(pos).unwrap();
+        let size = info.size();
 
-        if verified_chunks.contains(&info.digest) {
+        if verified_chunks.lock().unwrap().contains(&info.digest) {
             continue; // already verified
         }
 
-        if corrupt_chunks.contains(&info.digest) {
+        if corrupt_chunks.lock().unwrap().contains(&info.digest) {
             let digest_str = proxmox::tools::digest_to_hex(&info.digest);
             worker.log(format!("chunk {} was marked as corrupt", digest_str));
-            errors += 1;
+            errors.fetch_add(1, Ordering::SeqCst);
             continue;
         }
 
-        let chunk = match datastore.load_chunk(&info.digest) {
+        match datastore.load_chunk(&info.digest) {
             Err(err) => {
-                corrupt_chunks.insert(info.digest);
+                corrupt_chunks.lock().unwrap().insert(info.digest);
                 worker.log(format!("can't verify chunk, load failed - {}", err));
-                errors += 1;
+                errors.fetch_add(1, Ordering::SeqCst);
+                rename_corrupted_chunk(datastore.clone(), &info.digest, worker.clone());
                 continue;
-            },
-            Ok(chunk) => chunk,
-        };
-
-        let chunk_crypt_mode = match chunk.crypt_mode() {
-            Err(err) => {
-                corrupt_chunks.insert(info.digest);
-                worker.log(format!("can't verify chunk, unknown CryptMode - {}", err));
-                errors += 1;
-                continue;
-            },
-            Ok(mode) => mode,
-        };
-
-        if chunk_crypt_mode != crypt_mode {
-            worker.log(format!(
-                "chunk CryptMode {:?} does not match index CryptMode {:?}",
-                chunk_crypt_mode,
-                crypt_mode
-            ));
-            errors += 1;
-        }
-
-        let size = info.range.end - info.range.start;
-
-        if let Err(err) = chunk.verify_unencrypted(size as usize, &info.digest) {
-            corrupt_chunks.insert(info.digest);
-            worker.log(format!("{}", err));
-            errors += 1;
-        } else {
-            verified_chunks.insert(info.digest);
+            }
+            Ok(chunk) => {
+                read_bytes += chunk.raw_size();
+                decoder_pool.send((chunk, info.digest, size))?;
+                decoded_bytes += size;
+            }
         }
     }
 
-    if errors > 0 {
+    decoder_pool.complete()?;
+
+    let elapsed = start_time.elapsed().as_secs_f64();
+
+    let read_bytes_mib = (read_bytes as f64)/(1024.0*1024.0);
+    let decoded_bytes_mib = (decoded_bytes as f64)/(1024.0*1024.0);
+
+    let read_speed = read_bytes_mib/elapsed;
+    let decode_speed = decoded_bytes_mib/elapsed;
+
+    let error_count = errors.load(Ordering::SeqCst);
+
+    worker.log(format!("  verified {:.2}/{:.2} MiB in {:.2} seconds, speed {:.2}/{:.2} MiB/s ({} errors)",
+                       read_bytes_mib, decoded_bytes_mib, elapsed, read_speed, decode_speed, error_count));
+
+    if errors.load(Ordering::SeqCst) > 0 {
         bail!("chunks could not be verified");
     }
 
@@ -111,12 +190,12 @@ fn verify_index_chunks(
 }
 
 fn verify_fixed_index(
-    datastore: &DataStore,
+    datastore: Arc<DataStore>,
     backup_dir: &BackupDir,
     info: &FileInfo,
-    verified_chunks: &mut HashSet<[u8;32]>,
-    corrupt_chunks: &mut HashSet<[u8;32]>,
-    worker: &WorkerTask,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    worker: Arc<WorkerTask>,
 ) -> Result<(), Error> {
 
     let mut path = backup_dir.relative_path();
@@ -137,12 +216,12 @@ fn verify_fixed_index(
 }
 
 fn verify_dynamic_index(
-    datastore: &DataStore,
+    datastore: Arc<DataStore>,
     backup_dir: &BackupDir,
     info: &FileInfo,
-    verified_chunks: &mut HashSet<[u8;32]>,
-    corrupt_chunks: &mut HashSet<[u8;32]>,
-    worker: &WorkerTask,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    worker: Arc<WorkerTask>,
 ) -> Result<(), Error> {
 
     let mut path = backup_dir.relative_path();
@@ -172,11 +251,11 @@ fn verify_dynamic_index(
 /// - Ok(false) if there were verification errors
 /// - Err(_) if task was aborted
 pub fn verify_backup_dir(
-    datastore: &DataStore,
+    datastore: Arc<DataStore>,
     backup_dir: &BackupDir,
-    verified_chunks: &mut HashSet<[u8;32]>,
-    corrupt_chunks: &mut HashSet<[u8;32]>,
-    worker: &WorkerTask
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    worker: Arc<WorkerTask>
 ) -> Result<bool, Error> {
 
     let mut manifest = match datastore.load_manifest(&backup_dir) {
@@ -191,52 +270,52 @@ pub fn verify_backup_dir(
 
     let mut error_count = 0;
 
-    let mut verify_result = "ok";
+    let mut verify_result = VerifyState::Ok;
     for info in manifest.files() {
         let result = proxmox::try_block!({
             worker.log(format!("  check {}", info.filename));
             match archive_type(&info.filename)? {
                 ArchiveType::FixedIndex =>
                     verify_fixed_index(
-                        &datastore,
+                        datastore.clone(),
                         &backup_dir,
                         info,
-                        verified_chunks,
-                        corrupt_chunks,
-                        worker
+                        verified_chunks.clone(),
+                        corrupt_chunks.clone(),
+                        worker.clone(),
                     ),
                 ArchiveType::DynamicIndex =>
                     verify_dynamic_index(
-                        &datastore,
+                        datastore.clone(),
                         &backup_dir,
                         info,
-                        verified_chunks,
-                        corrupt_chunks,
-                        worker
+                        verified_chunks.clone(),
+                        corrupt_chunks.clone(),
+                        worker.clone(),
                     ),
-                ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info),
+                ArchiveType::Blob => verify_blob(datastore.clone(), &backup_dir, info),
             }
         });
 
         worker.fail_on_abort()?;
+        crate::tools::fail_on_shutdown()?;
 
         if let Err(err) = result {
             worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err));
             error_count += 1;
-            verify_result = "failed";
+            verify_result = VerifyState::Failed;
         }
 
     }
 
     let verify_state = SnapshotVerifyState {
-        state: verify_result.to_string(),
+        state: verify_result,
         upid: worker.upid().clone(),
     };
     manifest.unprotected["verify_state"] = serde_json::to_value(verify_state)?;
     datastore.store_manifest(&backup_dir, serde_json::to_value(manifest)?)
         .map_err(|err| format_err!("unable to store manifest blob - {}", err))?;
 
-
     Ok(error_count == 0)
 }
 
@@ -245,32 +324,45 @@ pub fn verify_backup_dir(
 /// Errors are logged to the worker log.
 ///
 /// Returns
-/// - Ok(failed_dirs) where failed_dirs had verification errors
+/// - Ok((count, failed_dirs)) where failed_dirs had verification errors
 /// - Err(_) if task was aborted
-pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<Vec<String>, Error> {
+pub fn verify_backup_group(
+    datastore: Arc<DataStore>,
+    group: &BackupGroup,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    progress: Option<(usize, usize)>, // (done, snapshot_count)
+    worker: Arc<WorkerTask>,
+) -> Result<(usize, Vec<String>), Error> {
 
     let mut errors = Vec::new();
     let mut list = match group.list_backups(&datastore.base_path()) {
         Ok(list) => list,
         Err(err) => {
             worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err));
-            return Ok(errors);
+            return Ok((0, errors));
         }
     };
 
     worker.log(format!("verify group {}:{}", datastore.name(), group));
 
-    let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB)
-    let mut corrupt_chunks = HashSet::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones
+    let (done, snapshot_count) = progress.unwrap_or((0, list.len()));
 
+    let mut count = 0;
     BackupInfo::sort_list(&mut list, false); // newest first
     for info in list {
-        if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, &mut corrupt_chunks, worker)?{
+        count += 1;
+        if !verify_backup_dir(datastore.clone(), &info.backup_dir, verified_chunks.clone(), corrupt_chunks.clone(), worker.clone())?{
             errors.push(info.backup_dir.to_string());
         }
+        if snapshot_count != 0 {
+            let pos = done + count;
+            let percentage = ((pos as f64) * 100.0)/(snapshot_count as f64);
+            worker.log(format!("percentage done: {:.2}% ({} of {} snapshots)", percentage, pos, snapshot_count));
+        }
     }
 
-    Ok(errors)
+    Ok((count, errors))
 }
 
 /// Verify all backups inside a datastore
@@ -280,12 +372,15 @@ pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &
 /// Returns
 /// - Ok(failed_dirs) where failed_dirs had verification errors
 /// - Err(_) if task was aborted
-pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<Vec<String>, Error> {
+pub fn verify_all_backups(datastore: Arc<DataStore>, worker: Arc<WorkerTask>) -> Result<Vec<String>, Error> {
 
     let mut errors = Vec::new();
 
     let mut list = match BackupGroup::list_groups(&datastore.base_path()) {
-        Ok(list) => list,
+        Ok(list) => list
+            .into_iter()
+            .filter(|group| !(group.backup_type() == "host" && group.backup_id() == "benchmark"))
+            .collect::<Vec<BackupGroup>>(),
         Err(err) => {
             worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
             return Ok(errors);
@@ -294,11 +389,32 @@ pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<
 
     list.sort_unstable();
 
-    worker.log(format!("verify datastore {}", datastore.name()));
+    let mut snapshot_count = 0;
+    for group in list.iter() {
+        snapshot_count += group.list_backups(&datastore.base_path())?.len();
+    }
 
+    // start with 16384 chunks (up to 65GB)
+    let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*16)));
+
+    // start with 64 chunks since we assume there are few corrupt ones
+    let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
+
+    worker.log(format!("verify datastore {} ({} snapshots)", datastore.name(), snapshot_count));
+
+    let mut done = 0;
     for group in list {
-        let mut group_errors = verify_backup_group(datastore, &group, worker)?;
+        let (count, mut group_errors) = verify_backup_group(
+            datastore.clone(),
+            &group,
+            verified_chunks.clone(),
+            corrupt_chunks.clone(),
+            Some((done, snapshot_count)),
+            worker.clone(),
+        )?;
         errors.append(&mut group_errors);
+
+        done += count;
     }
 
     Ok(errors)

src/bin/proxmox-backup-client.rs

@@ -8,7 +8,6 @@ use std::sync::{Arc, Mutex};
 use std::task::Context;
 
 use anyhow::{bail, format_err, Error};
-use chrono::{Local, DateTime, Utc, TimeZone};
 use futures::future::FutureExt;
 use futures::stream::{StreamExt, TryStreamExt};
 use serde_json::{json, Value};
@@ -16,11 +15,20 @@ use tokio::sync::mpsc;
 use xdg::BaseDirectories;
 
 use pathpatterns::{MatchEntry, MatchType, PatternFlag};
-use proxmox::tools::fs::{file_get_contents, file_get_json, replace_file, CreateOptions, image_size};
-use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment};
-use proxmox::api::schema::*;
-use proxmox::api::cli::*;
-use proxmox::api::api;
+use proxmox::{
+    tools::{
+        time::{strftime_local, epoch_i64},
+        fs::{file_get_contents, file_get_json, replace_file, CreateOptions, image_size},
+    },
+    api::{
+        api,
+        ApiHandler,
+        ApiMethod,
+        RpcEnvironment,
+        schema::*,
+        cli::*,
+    },
+};
 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
 
 use proxmox_backup::tools;
@@ -184,7 +192,7 @@ pub fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<
     result
 }
 
-fn connect(server: &str, userid: &Userid) -> Result<HttpClient, Error> {
+fn connect(server: &str, port: u16, userid: &Userid) -> Result<HttpClient, Error> {
 
     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
 
@@ -203,7 +211,7 @@ fn connect(server: &str, userid: &Userid) -> Result<HttpClient, Error> {
         .fingerprint_cache(true)
         .ticket_cache(true);
 
-    HttpClient::new(server, userid, options)
+    HttpClient::new(server, port, userid, options)
 }
 
 async fn view_task_result(
@@ -246,7 +254,7 @@ pub async fn api_datastore_latest_snapshot(
     client: &HttpClient,
     store: &str,
     group: BackupGroup,
-) -> Result<(String, String, DateTime<Utc>), Error> {
+) -> Result<(String, String, i64), Error> {
 
     let list = api_datastore_list_snapshots(client, store, Some(group.clone())).await?;
     let mut list: Vec<SnapshotListItem> = serde_json::from_value(list)?;
@@ -257,7 +265,7 @@ pub async fn api_datastore_latest_snapshot(
 
     list.sort_unstable_by(|a, b| b.backup_time.cmp(&a.backup_time));
 
-    let backup_time = Utc.timestamp(list[0].backup_time, 0);
+    let backup_time = list[0].backup_time;
 
     Ok((group.backup_type().to_owned(), group.backup_id().to_owned(), backup_time))
 }
@@ -357,7 +365,7 @@ async fn list_backup_groups(param: Value) -> Result<Value, Error> {
 
     let repo = extract_repository_from_value(&param)?;
 
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
 
     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 
@@ -373,7 +381,7 @@ async fn list_backup_groups(param: Value) -> Result<Value, Error> {
 
     let render_last_backup = |_v: &Value, record: &Value| -> Result<String, Error> {
         let item: GroupListItem = serde_json::from_value(record.to_owned())?;
-        let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup);
+        let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.last_backup)?;
         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
     };
 
@@ -430,7 +438,7 @@ async fn list_snapshots(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
 
     let group: Option<BackupGroup> = if let Some(path) = param["group"].as_str() {
         Some(path.parse()?)
@@ -444,7 +452,7 @@ async fn list_snapshots(param: Value) -> Result<Value, Error> {
 
     let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
         let item: SnapshotListItem = serde_json::from_value(record.to_owned())?;
-        let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time);
+        let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
     };
 
@@ -495,14 +503,14 @@ async fn forget_snapshots(param: Value) -> Result<Value, Error> {
     let path = tools::required_string_param(&param, "snapshot")?;
     let snapshot: BackupDir = path.parse()?;
 
-    let mut client = connect(repo.host(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.user())?;
 
     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 
     let result = client.delete(&path, Some(json!({
         "backup-type": snapshot.group().backup_type(),
         "backup-id": snapshot.group().backup_id(),
-        "backup-time": snapshot.backup_time().timestamp(),
+        "backup-time": snapshot.backup_time(),
     }))).await?;
 
     record_repository(&repo);
@@ -525,7 +533,7 @@ async fn api_login(param: Value) -> Result<Value, Error> {
 
     let repo = extract_repository_from_value(&param)?;
 
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
     client.login().await?;
 
     record_repository(&repo);
@@ -582,7 +590,7 @@ async fn api_version(param: Value) -> Result<(), Error> {
 
     let repo = extract_repository_from_value(&param);
     if let Ok(repo) = repo {
-        let client = connect(repo.host(), repo.user())?;
+        let client = connect(repo.host(), repo.port(), repo.user())?;
 
         match client.get("api2/json/version", None).await {
             Ok(mut result) => version_info["server"] = result["data"].take(),
@@ -632,14 +640,14 @@ async fn list_snapshot_files(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
 
     let path = format!("api2/json/admin/datastore/{}/files", repo.store());
 
     let mut result = client.get(&path, Some(json!({
         "backup-type": snapshot.group().backup_type(),
         "backup-id": snapshot.group().backup_id(),
-        "backup-time": snapshot.backup_time().timestamp(),
+        "backup-time": snapshot.backup_time(),
     }))).await?;
 
     record_repository(&repo);
@@ -676,7 +684,7 @@ async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let mut client = connect(repo.host(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.user())?;
 
     let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
 
@@ -986,18 +994,18 @@ async fn create_backup(
         }
     }
 
-    let backup_time = Utc.timestamp(backup_time_opt.unwrap_or_else(|| Utc::now().timestamp()), 0);
+    let backup_time = backup_time_opt.unwrap_or_else(|| epoch_i64());
 
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
     record_repository(&repo);
 
-    println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time));
+    println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time)?);
 
     println!("Client name: {}", proxmox::tools::nodename());
 
-    let start_time = Local::now();
+    let start_time = std::time::Instant::now();
 
-    println!("Starting protocol: {}", start_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
+    println!("Starting backup protocol: {}", strftime_local("%c", epoch_i64())?);
 
     let (crypt_config, rsa_encrypted_key) = match keydata {
         None => (None, None),
@@ -1026,6 +1034,7 @@ async fn create_backup(
         &backup_id,
         backup_time,
         verbose,
+        false
     ).await?;
 
     let previous_manifest = if let Ok(previous_manifest) = client.download_previous_manifest().await {
@@ -1034,7 +1043,7 @@ async fn create_backup(
         None
     };
 
-    let snapshot = BackupDir::new(backup_type, backup_id, backup_time.timestamp());
+    let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
     let mut manifest = BackupManifest::new(snapshot);
 
     let mut catalog = None;
@@ -1149,11 +1158,11 @@ async fn create_backup(
 
     client.finish().await?;
 
-    let end_time = Local::now();
-    let elapsed = end_time.signed_duration_since(start_time);
-    println!("Duration: {}", elapsed);
+    let end_time = std::time::Instant::now();
+    let elapsed = end_time.duration_since(start_time);
+    println!("Duration: {:.2}s", elapsed.as_secs_f64());
 
-    println!("End Time: {}", end_time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false));
+    println!("End Time: {}", strftime_local("%c", epoch_i64())?);
 
     Ok(Value::Null)
 }
@@ -1290,7 +1299,7 @@ async fn restore(param: Value) -> Result<Value, Error> {
 
     let archive_name = tools::required_string_param(&param, "archive-name")?;
 
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
 
     record_repository(&repo);
 
@@ -1463,7 +1472,7 @@ async fn upload_log(param: Value) -> Result<Value, Error> {
     let snapshot = tools::required_string_param(&param, "snapshot")?;
     let snapshot: BackupDir = snapshot.parse()?;
 
-    let mut client = connect(repo.host(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.user())?;
 
     let (keydata, crypt_mode) = keyfile_parameters(&param)?;
 
@@ -1491,7 +1500,7 @@ async fn upload_log(param: Value) -> Result<Value, Error> {
     let args = json!({
         "backup-type": snapshot.group().backup_type(),
         "backup-id":  snapshot.group().backup_id(),
-        "backup-time": snapshot.backup_time().timestamp(),
+        "backup-time": snapshot.backup_time(),
     });
 
     let body = hyper::Body::from(raw_data);
@@ -1534,7 +1543,7 @@ fn prune<'a>(
 async fn prune_async(mut param: Value) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
 
-    let mut client = connect(repo.host(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.user())?;
 
     let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
 
@@ -1559,7 +1568,7 @@ async fn prune_async(mut param: Value) -> Result<Value, Error> {
 
     let render_snapshot_path = |_v: &Value, record: &Value| -> Result<String, Error> {
         let item: PruneListItem = serde_json::from_value(record.to_owned())?;
-        let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time);
+        let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
         Ok(snapshot.relative_path().to_str().unwrap().to_owned())
     };
 
@@ -1617,7 +1626,7 @@ async fn status(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
 
     let path = format!("api2/json/admin/datastore/{}/status", repo.store());
 
@@ -1662,7 +1671,7 @@ async fn try_get(repo: &BackupRepository, url: &str) -> Value {
         .fingerprint_cache(true)
         .ticket_cache(true);
 
-    let client = match HttpClient::new(repo.host(), repo.user(), options) {
+    let client = match HttpClient::new(repo.host(), repo.port(), repo.user(), options) {
         Ok(v) => v,
         _ => return Value::Null,
     };
@@ -1751,8 +1760,9 @@ async fn complete_backup_snapshot_do(param: &HashMap<String, String>) -> Vec<Str
             if let (Some(backup_id), Some(backup_type), Some(backup_time)) =
                 (item["backup-id"].as_str(), item["backup-type"].as_str(), item["backup-time"].as_i64())
             {
-                let snapshot = BackupDir::new(backup_type, backup_id, backup_time);
-                result.push(snapshot.relative_path().to_str().unwrap().to_owned());
+                if let Ok(snapshot) = BackupDir::new(backup_type, backup_id, backup_time) {
+                    result.push(snapshot.relative_path().to_str().unwrap().to_owned());
+                }
             }
         }
     }
@@ -1786,7 +1796,7 @@ async fn complete_server_file_name_do(param: &HashMap<String, String>) -> Vec<St
     let query = tools::json_object_to_query(json!({
         "backup-type": snapshot.group().backup_type(),
         "backup-id": snapshot.group().backup_id(),
-        "backup-time": snapshot.backup_time().timestamp(),
+        "backup-time": snapshot.backup_time(),
     })).unwrap();
 
     let path = format!("api2/json/admin/datastore/{}/files?{}", repo.store(), query);

src/bin/proxmox-backup-manager.rs

@@ -62,10 +62,10 @@ fn connect() -> Result<HttpClient, Error> {
         let ticket = Ticket::new("PBS", Userid::root_userid())?
             .sign(private_auth_key(), None)?;
         options = options.password(Some(ticket));
-        HttpClient::new("localhost", Userid::root_userid(), options)?
+        HttpClient::new("localhost", 8007, Userid::root_userid(), options)?
     } else {
         options = options.ticket_cache(true).interactive(true);
-        HttpClient::new("localhost", Userid::root_userid(), options)?
+        HttpClient::new("localhost", 8007, Userid::root_userid(), options)?
     };
 
     Ok(client)
@@ -410,6 +410,7 @@ pub fn complete_remote_datastore_name(_arg: &str, param: &HashMap<String, String
 
         let client = HttpClient::new(
             &remote.host,
+            remote.port.unwrap_or(8007),
             &remote.userid,
             options,
         )?;

src/bin/proxmox-backup-proxy.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::sync::{Arc};
 use std::path::{Path, PathBuf};
 
 use anyhow::{bail, format_err, Error};
@@ -13,7 +13,7 @@ use proxmox_backup::api2::types::Userid;
 use proxmox_backup::configdir;
 use proxmox_backup::buildcfg;
 use proxmox_backup::server;
-use proxmox_backup::tools::{daemon, epoch_now, epoch_now_u64};
+use proxmox_backup::tools::daemon;
 use proxmox_backup::server::{ApiConfig, rest::*};
 use proxmox_backup::auth_helpers::*;
 use proxmox_backup::tools::disks::{ DiskManage, zfs_pool_stats };
@@ -53,6 +53,7 @@ async fn run() -> Result<(), Error> {
     config.add_alias("extjs", "/usr/share/javascript/extjs");
     config.add_alias("fontawesome", "/usr/share/fonts-font-awesome");
     config.add_alias("xtermjs", "/usr/share/pve-xtermjs");
+    config.add_alias("locale", "/usr/share/pbs-i18n");
     config.add_alias("widgettoolkit", "/usr/share/javascript/proxmox-widget-toolkit");
     config.add_alias("css", "/usr/share/javascript/proxmox-backup/css");
     config.add_alias("docs", "/usr/share/doc/proxmox-backup/html");
@@ -86,8 +87,6 @@ async fn run() -> Result<(), Error> {
                     let acceptor = Arc::clone(&acceptor);
                     async move {
                         sock.set_nodelay(true).unwrap();
-                        sock.set_send_buffer_size(1024*1024).unwrap();
-                        sock.set_recv_buffer_size(1024*1024).unwrap();
                         Ok(tokio_openssl::accept(&acceptor, sock)
                             .await
                             .ok() // handshake errors aren't be fatal, so return None to filter
@@ -145,11 +144,12 @@ fn start_task_scheduler() {
     tokio::spawn(task.map(|_| ()));
 }
 
-use std::time:: {Instant, Duration};
+use std::time::{SystemTime, Instant, Duration, UNIX_EPOCH};
 
 fn next_minute() -> Result<Instant, Error> {
-    let epoch_now = epoch_now()?;
-    let epoch_next = Duration::from_secs((epoch_now.as_secs()/60 + 1)*60);
+    let now = SystemTime::now();
+    let epoch_now = now.duration_since(UNIX_EPOCH)?;
+    let epoch_next = Duration::from_secs((epoch_now.as_secs()/60  + 1)*60);
     Ok(Instant::now() + epoch_next - epoch_now)
 }
 
@@ -196,45 +196,21 @@ async fn schedule_tasks() -> Result<(), Error> {
 
     schedule_datastore_garbage_collection().await;
     schedule_datastore_prune().await;
+    schedule_datastore_verification().await;
     schedule_datastore_sync_jobs().await;
+    schedule_task_log_rotate().await;
 
     Ok(())
 }
 
-fn lookup_last_worker(worker_type: &str, worker_id: &str) -> Result<Option<server::UPID>, Error> {
-
-    let list = proxmox_backup::server::read_task_list()?;
-
-    let mut last: Option<&server::UPID> = None;
-
-    for entry in list.iter() {
-        if entry.upid.worker_type == worker_type {
-            if let Some(ref id) = entry.upid.worker_id {
-                if id == worker_id {
-                    match last {
-                        Some(ref upid) => {
-                            if upid.starttime < entry.upid.starttime {
-                                last = Some(&entry.upid)
-                            }
-                        }
-                        None => {
-                            last = Some(&entry.upid)
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    Ok(last.cloned())
-}
-
-
 async fn schedule_datastore_garbage_collection() {
 
     use proxmox_backup::backup::DataStore;
     use proxmox_backup::server::{UPID, WorkerTask};
-    use proxmox_backup::config::datastore::{self, DataStoreConfig};
+    use proxmox_backup::config::{
+        jobstate::{self, Job},
+        datastore::{self, DataStoreConfig}
+    };
     use proxmox_backup::tools::systemd::time::{
         parse_calendar_event, compute_next_event};
 
@@ -290,33 +266,33 @@ async fn schedule_datastore_garbage_collection() {
                 }
             }
         } else {
-            match lookup_last_worker(worker_type, &store) {
-                Ok(Some(upid)) => upid.starttime,
-                Ok(None) => 0,
+            match jobstate::last_run_time(worker_type, &store) {
+                Ok(time) => time,
                 Err(err) => {
-                    eprintln!("lookup_last_job_start failed: {}", err);
+                    eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
                     continue;
                 }
             }
         };
 
         let next = match compute_next_event(&event, last, false) {
-            Ok(next) => next,
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
             Err(err) => {
                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
                 continue;
             }
         };
 
-        let now = match epoch_now_u64() {
-            Ok(epoch_now) => epoch_now as i64,
-            Err(err) => {
-                eprintln!("query system time failed - {}", err);
-                continue;
-            }
-        };
+        let now = proxmox::tools::time::epoch_i64();
+
         if next > now  { continue; }
 
+        let mut job = match Job::new(worker_type, &store) {
+            Ok(job) => job,
+            Err(_) => continue, // could not get lock
+        };
+
         let store2 = store.clone();
 
         if let Err(err) = WorkerTask::new_thread(
@@ -325,9 +301,20 @@ async fn schedule_datastore_garbage_collection() {
             Userid::backup_userid().clone(),
             false,
             move |worker| {
+                job.start(&worker.upid().to_string())?;
+
                 worker.log(format!("starting garbage collection on store {}", store));
                 worker.log(format!("task triggered by schedule '{}'", event_str));
-                datastore.garbage_collection(&worker)
+
+                let result = datastore.garbage_collection(&worker);
+
+                let status = worker.create_state(&result);
+
+                if let Err(err) = job.finish(status) {
+                    eprintln!("could not finish job state for {}: {}", worker_type, err);
+                }
+
+                result
             }
         ) {
             eprintln!("unable to start garbage collection on store {} - {}", store2, err);
@@ -338,9 +325,12 @@ async fn schedule_datastore_garbage_collection() {
 async fn schedule_datastore_prune() {
 
     use proxmox_backup::backup::{
-        PruneOptions, DataStore, BackupGroup, BackupDir, compute_prune_info};
+        PruneOptions, DataStore, BackupGroup, compute_prune_info};
     use proxmox_backup::server::{WorkerTask};
-    use proxmox_backup::config::datastore::{self, DataStoreConfig};
+    use proxmox_backup::config::{
+        jobstate::{self, Job},
+        datastore::{self, DataStoreConfig}
+    };
     use proxmox_backup::tools::systemd::time::{
         parse_calendar_event, compute_next_event};
 
@@ -397,37 +387,32 @@ async fn schedule_datastore_prune() {
 
         let worker_type = "prune";
 
-        let last = match lookup_last_worker(worker_type, &store) {
-            Ok(Some(upid)) => {
-                if proxmox_backup::server::worker_is_active_local(&upid) {
-                    continue;
-                }
-                upid.starttime
-            }
-            Ok(None) => 0,
+        let last = match jobstate::last_run_time(worker_type, &store) {
+            Ok(time) => time,
             Err(err) => {
-                eprintln!("lookup_last_job_start failed: {}", err);
+                eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
                 continue;
             }
         };
 
         let next = match compute_next_event(&event, last, false) {
-            Ok(next) => next,
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
             Err(err) => {
                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
                 continue;
             }
         };
 
-        let now = match epoch_now_u64() {
-            Ok(epoch_now) => epoch_now as i64,
-            Err(err) => {
-                eprintln!("query system time failed - {}", err);
-                continue;
-            }
-        };
+        let now = proxmox::tools::time::epoch_i64();
+
         if next > now  { continue; }
 
+        let mut job = match Job::new(worker_type, &store) {
+            Ok(job) => job,
+            Err(_) => continue, // could not get lock
+        };
+
         let store2 = store.clone();
 
         if let Err(err) = WorkerTask::new_thread(
@@ -436,35 +421,47 @@ async fn schedule_datastore_prune() {
             Userid::backup_userid().clone(),
             false,
             move |worker| {
-                worker.log(format!("Starting datastore prune on store \"{}\"", store));
-                worker.log(format!("task triggered by schedule '{}'", event_str));
-                worker.log(format!("retention options: {}", prune_options.cli_options_string()));
 
-                let base_path = datastore.base_path();
+                job.start(&worker.upid().to_string())?;
 
-                let groups = BackupGroup::list_groups(&base_path)?;
-                for group in groups {
-                    let list = group.list_backups(&base_path)?;
-                    let mut prune_info = compute_prune_info(list, &prune_options)?;
-                    prune_info.reverse(); // delete older snapshots first
+                let result = try_block!({
 
-                    worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
-                                       store, group.backup_type(), group.backup_id()));
+                    worker.log(format!("Starting datastore prune on store \"{}\"", store));
+                    worker.log(format!("task triggered by schedule '{}'", event_str));
+                    worker.log(format!("retention options: {}", prune_options.cli_options_string()));
 
-                    for (info, keep) in prune_info {
-                        worker.log(format!(
-                            "{} {}/{}/{}",
-                            if keep { "keep" } else { "remove" },
-                            group.backup_type(), group.backup_id(),
-                            BackupDir::backup_time_to_string(info.backup_dir.backup_time())));
+                    let base_path = datastore.base_path();
 
-                        if !keep {
-                            datastore.remove_backup_dir(&info.backup_dir, true)?;
+                    let groups = BackupGroup::list_groups(&base_path)?;
+                    for group in groups {
+                        let list = group.list_backups(&base_path)?;
+                        let mut prune_info = compute_prune_info(list, &prune_options)?;
+                        prune_info.reverse(); // delete older snapshots first
+
+                        worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
+                                store, group.backup_type(), group.backup_id()));
+
+                        for (info, keep) in prune_info {
+                            worker.log(format!(
+                                    "{} {}/{}/{}",
+                                    if keep { "keep" } else { "remove" },
+                                    group.backup_type(), group.backup_id(),
+                                    info.backup_dir.backup_time_string()));
+                            if !keep {
+                                datastore.remove_backup_dir(&info.backup_dir, true)?;
+                            }
                         }
                     }
+                    Ok(())
+                });
+
+                let status = worker.create_state(&result);
+
+                if let Err(err) = job.finish(status) {
+                    eprintln!("could not finish job state for {}: {}", worker_type, err);
                 }
 
-                Ok(())
+                result
             }
         ) {
             eprintln!("unable to start datastore prune on store {} - {}", store2, err);
@@ -472,6 +469,120 @@ async fn schedule_datastore_prune() {
     }
 }
 
+async fn schedule_datastore_verification() {
+    use proxmox_backup::backup::{DataStore, verify_all_backups};
+    use proxmox_backup::server::{WorkerTask};
+    use proxmox_backup::config::{
+        jobstate::{self, Job},
+        datastore::{self, DataStoreConfig}
+    };
+    use proxmox_backup::tools::systemd::time::{
+        parse_calendar_event, compute_next_event};
+
+    let config = match datastore::config() {
+        Err(err) => {
+            eprintln!("unable to read datastore config - {}", err);
+            return;
+        }
+        Ok((config, _digest)) => config,
+    };
+
+    for (store, (_, store_config)) in config.sections {
+        let datastore = match DataStore::lookup_datastore(&store) {
+            Ok(datastore) => datastore,
+            Err(err) => {
+                eprintln!("lookup_datastore failed - {}", err);
+                continue;
+            }
+        };
+
+        let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
+            Ok(c) => c,
+            Err(err) => {
+                eprintln!("datastore config from_value failed - {}", err);
+                continue;
+            }
+        };
+
+        let event_str = match store_config.verify_schedule {
+            Some(event_str) => event_str,
+            None => continue,
+        };
+
+        let event = match parse_calendar_event(&event_str) {
+            Ok(event) => event,
+            Err(err) => {
+                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
+                continue;
+            }
+        };
+
+        let worker_type = "verify";
+
+        let last = match jobstate::last_run_time(worker_type, &store) {
+            Ok(time) => time,
+            Err(err) => {
+                eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
+                continue;
+            }
+        };
+
+        let next = match compute_next_event(&event, last, false) {
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
+            Err(err) => {
+                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
+                continue;
+            }
+        };
+
+        let now = proxmox::tools::time::epoch_i64();
+
+        if next > now { continue; }
+
+        let mut job = match Job::new(worker_type, &store) {
+            Ok(job) => job,
+            Err(_) => continue, // could not get lock
+        };
+
+        let worker_id = store.clone();
+        let store2 = store.clone();
+        if let Err(err) = WorkerTask::new_thread(
+            worker_type,
+            Some(worker_id),
+            Userid::backup_userid().clone(),
+            false,
+            move |worker| {
+                job.start(&worker.upid().to_string())?;
+                worker.log(format!("starting verification on store {}", store2));
+                worker.log(format!("task triggered by schedule '{}'", event_str));
+                let result = try_block!({
+                    let failed_dirs = verify_all_backups(datastore, worker.clone())?;
+                    if failed_dirs.len() > 0 {
+                        worker.log("Failed to verify following snapshots:");
+                        for dir in failed_dirs {
+                            worker.log(format!("\t{}", dir));
+                        }
+                        Err(format_err!("verification failed - please check the log for details"))
+                    } else {
+                        Ok(())
+                    }
+                });
+
+                let status = worker.create_state(&result);
+
+                if let Err(err) = job.finish(status) {
+                    eprintln!("could not finish job state for {}: {}", worker_type, err);
+                }
+
+                result
+            },
+        ) {
+            eprintln!("unable to start verification on store {} - {}", store, err);
+        }
+    }
+}
+
 async fn schedule_datastore_sync_jobs() {
 
     use proxmox_backup::{
@@ -520,20 +631,16 @@ async fn schedule_datastore_sync_jobs() {
         };
 
         let next = match compute_next_event(&event, last, false) {
-            Ok(next) => next,
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
             Err(err) => {
                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
                 continue;
             }
         };
 
-        let now = match epoch_now_u64() {
-            Ok(epoch_now) => epoch_now as i64,
-            Err(err) => {
-                eprintln!("query system time failed - {}", err);
-                continue;
-            }
-        };
+        let now = proxmox::tools::time::epoch_i64();
+
         if next > now  { continue; }
 
         let job = match Job::new(worker_type, &job_id) {
@@ -549,6 +656,101 @@ async fn schedule_datastore_sync_jobs() {
     }
 }
 
+async fn schedule_task_log_rotate() {
+    use proxmox_backup::{
+        config::jobstate::{self, Job},
+        server::rotate_task_log_archive,
+    };
+    use proxmox_backup::server::WorkerTask;
+    use proxmox_backup::tools::systemd::time::{
+        parse_calendar_event, compute_next_event};
+
+    let worker_type = "logrotate";
+    let job_id = "task-archive";
+
+    let last = match jobstate::last_run_time(worker_type, job_id) {
+        Ok(time) => time,
+        Err(err) => {
+            eprintln!("could not get last run time of task log archive rotation: {}", err);
+            return;
+        }
+    };
+
+    // schedule daily at 00:00 like normal logrotate
+    let schedule = "00:00";
+
+    let event = match parse_calendar_event(schedule) {
+        Ok(event) => event,
+        Err(err) => {
+            // should not happen?
+            eprintln!("unable to parse schedule '{}' - {}", schedule, err);
+            return;
+        }
+    };
+
+    let next = match compute_next_event(&event, last, false) {
+        Ok(Some(next)) => next,
+        Ok(None) => return,
+        Err(err) => {
+            eprintln!("compute_next_event for '{}' failed - {}", schedule, err);
+            return;
+        }
+    };
+
+    let now = proxmox::tools::time::epoch_i64();
+
+    if next > now {
+        // if we never ran the rotation, schedule instantly
+        match jobstate::JobState::load(worker_type, job_id) {
+            Ok(state) => match state {
+                jobstate::JobState::Created { .. } => {},
+                _ => return,
+            },
+            _ => return,
+        }
+    }
+
+    let mut job = match Job::new(worker_type, job_id) {
+        Ok(job) => job,
+        Err(_) => return, // could not get lock
+    };
+
+    if let Err(err) = WorkerTask::new_thread(
+        worker_type,
+        Some(job_id.to_string()),
+        Userid::backup_userid().clone(),
+        false,
+        move |worker| {
+            job.start(&worker.upid().to_string())?;
+            worker.log(format!("starting task log rotation"));
+            // one entry has normally about ~100-150 bytes
+            let max_size = 500000; // at least 5000 entries
+            let max_files = 20; // at least 100000 entries
+            let result = try_block!({
+                let has_rotated = rotate_task_log_archive(max_size, true, Some(max_files))?;
+                if has_rotated {
+                    worker.log(format!("task log archive was rotated"));
+                } else {
+                    worker.log(format!("task log archive was not rotated"));
+                }
+
+                Ok(())
+            });
+
+            let status = worker.create_state(&result);
+
+            if let Err(err) = job.finish(status) {
+                eprintln!("could not finish job state for {}: {}", worker_type, err);
+            }
+
+            result
+        },
+    ) {
+        eprintln!("unable to start task log rotation: {}", err);
+    }
+
+}
+
 async fn run_stat_generator() {
 
     let mut count = 0;

src/bin/proxmox_backup_client/benchmark.rs

@@ -3,7 +3,6 @@ use std::sync::Arc;
 
 use anyhow::{Error};
 use serde_json::Value;
-use chrono::{TimeZone, Utc};
 use serde::Serialize;
 
 use proxmox::api::{ApiMethod, RpcEnvironment};
@@ -22,6 +21,7 @@ use proxmox_backup::backup::{
     load_and_decrypt_key,
     CryptConfig,
     KeyDerivationConfig,
+    DataChunkBuilder,
 };
 
 use proxmox_backup::client::*;
@@ -61,6 +61,9 @@ struct Speed {
         "aes256_gcm": {
             type: Speed,
         },
+        "verify": {
+            type: Speed,
+        },
     },
 )]
 #[derive(Copy, Clone, Serialize)]
@@ -76,29 +79,34 @@ struct BenchmarkResult {
     decompress: Speed,
     /// AES256 GCM encryption speed
     aes256_gcm: Speed,
+    /// Verify speed
+    verify: Speed,
 }
 
-
 static BENCHMARK_RESULT_2020_TOP: BenchmarkResult =  BenchmarkResult {
     tls: Speed {
         speed: None,
-        top: 1_000_000.0 * 590.0, // TLS to localhost, AMD Ryzen 7 2700X
+        top: 1_000_000.0 * 690.0, // TLS to localhost, AMD Ryzen 7 2700X
     },
     sha256: Speed {
         speed: None,
-        top: 1_000_000.0 * 2120.0, // AMD Ryzen 7 2700X
+        top: 1_000_000.0 * 2022.0, // AMD Ryzen 7 2700X
     },
     compress: Speed {
         speed: None,
-        top: 1_000_000.0 * 2158.0, // AMD Ryzen 7 2700X
+        top: 1_000_000.0 * 752.0, // AMD Ryzen 7 2700X
     },
     decompress: Speed {
         speed: None,
-        top: 1_000_000.0 * 8062.0, // AMD Ryzen 7 2700X
+        top: 1_000_000.0 * 1198.0, // AMD Ryzen 7 2700X
     },
     aes256_gcm: Speed {
         speed: None,
-        top: 1_000_000.0 * 3803.0, // AMD Ryzen 7 2700X
+        top: 1_000_000.0 * 3645.0, // AMD Ryzen 7 2700X
+    },
+    verify: Speed {
+        speed: None,
+        top: 1_000_000.0 * 758.0, // AMD Ryzen 7 2700X
     },
 };
 
@@ -195,7 +203,10 @@ fn render_result(
         .column(ColumnConfig::new("decompress")
                 .header("ZStd level 1 decompression speed")
                 .right_align(false).renderer(render_speed))
-        .column(ColumnConfig::new("aes256_gcm")
+        .column(ColumnConfig::new("verify")
+                .header("Chunk verification speed")
+                .right_align(false).renderer(render_speed))
+       .column(ColumnConfig::new("aes256_gcm")
                 .header("AES256 GCM encryption speed")
                 .right_align(false).renderer(render_speed));
 
@@ -212,9 +223,9 @@ async fn test_upload_speed(
     verbose: bool,
 ) -> Result<(), Error> {
 
-    let backup_time = Utc.timestamp(Utc::now().timestamp(), 0);
+    let backup_time = proxmox::tools::time::epoch_i64();
 
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
     record_repository(&repo);
 
     if verbose { eprintln!("Connecting to backup server"); }
@@ -226,6 +237,7 @@ async fn test_upload_speed(
         "benchmark",
         backup_time,
         false,
+        true
     ).await?;
 
     if verbose { eprintln!("Start TLS speed test"); }
@@ -257,7 +269,17 @@ fn test_crypt_speed(
 
     let crypt_config = CryptConfig::new(testkey)?;
 
-    let random_data = proxmox::sys::linux::random_data(1024*1024)?;
+    //let random_data = proxmox::sys::linux::random_data(1024*1024)?;
+    let mut random_data = vec![];
+        // generate pseudo random byte sequence
+        for i in 0..256*1024 {
+            for j in 0..4 {
+                let byte = ((i >> (j<<3))&0xff) as u8;
+                random_data.push(byte);
+            }
+        }
+
+    assert_eq!(random_data.len(), 1024*1024);
 
     let start_time = std::time::Instant::now();
 
@@ -322,5 +344,23 @@ fn test_crypt_speed(
 
     eprintln!("AES256/GCM speed: {:.2} MB/s", speed/1_000_000_.0);
 
+
+    let start_time = std::time::Instant::now();
+
+    let (chunk, digest) = DataChunkBuilder::new(&random_data)
+        .compress(true)
+        .build()?;
+
+    let mut bytes = 0;
+    loop  {
+        chunk.verify_unencrypted(random_data.len(), &digest)?;
+        bytes += random_data.len();
+        if start_time.elapsed().as_micros() > 1_000_000 { break; }
+    }
+    let speed = (bytes as f64)/start_time.elapsed().as_secs_f64();
+    benchmark_result.verify.speed = Some(speed);
+
+    eprintln!("Verify speed: {:.2} MB/s", speed/1_000_000_.0);
+
     Ok(())
 }

src/bin/proxmox_backup_client/catalog.rs

@@ -79,7 +79,7 @@ async fn dump_catalog(param: Value) -> Result<Value, Error> {
         }
     };
 
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
 
     let client = BackupReader::start(
         client,
@@ -153,7 +153,7 @@ async fn dump_catalog(param: Value) -> Result<Value, Error> {
 /// Shell to interactively inspect and restore snapshots.
 async fn catalog_shell(param: Value) -> Result<(), Error> {
     let repo = extract_repository_from_value(&param)?;
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
     let path = tools::required_string_param(&param, "snapshot")?;
     let archive_name = tools::required_string_param(&param, "archive-name")?;
 

src/bin/proxmox_backup_client/key.rs

@@ -1,7 +1,8 @@
 use std::path::PathBuf;
+use std::io::Write;
+use std::process::{Stdio, Command};
 
 use anyhow::{bail, format_err, Error};
-use chrono::{Local, TimeZone};
 use serde::{Deserialize, Serialize};
 
 use proxmox::api::api;
@@ -14,6 +15,17 @@ use proxmox_backup::backup::{
 };
 use proxmox_backup::tools;
 
+#[api()]
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// Paperkey output format
+pub enum PaperkeyFormat {
+    /// Format as Utf8 text. Includes QR codes as ascii-art.
+    Text,
+    /// Format as Html. Includes QR codes as png images.
+    Html,
+}
+
 pub const DEFAULT_ENCRYPTION_KEY_FILE_NAME: &str = "encryption-key.json";
 pub const MASTER_PUBKEY_FILE_NAME: &str = "master-public.pem";
 
@@ -112,7 +124,7 @@ fn create(kdf: Option<Kdf>, path: Option<String>) -> Result<(), Error> {
 
     match kdf {
         Kdf::None => {
-            let created = Local.timestamp(Local::now().timestamp(), 0);
+            let created = proxmox::tools::time::epoch_i64();
 
             store_key_config(
                 &path,
@@ -180,7 +192,7 @@ fn change_passphrase(kdf: Option<Kdf>, path: Option<String>) -> Result<(), Error
 
     match kdf {
         Kdf::None => {
-            let modified = Local.timestamp(Local::now().timestamp(), 0);
+            let modified = proxmox::tools::time::epoch_i64();
 
             store_key_config(
                 &path,
@@ -262,6 +274,55 @@ fn create_master_key() -> Result<(), Error> {
     Ok(())
 }
 
+#[api(
+    input: {
+        properties: {
+            path: {
+                description: "Key file. Without this the default key's will be used.",
+                optional: true,
+            },
+            subject: {
+                description: "Include the specified subject as titel text.",
+                optional: true,
+            },
+            "output-format": {
+                type: PaperkeyFormat,
+                description: "Output format. Text or Html.",
+                optional: true,
+            },
+        },
+    },
+)]
+/// Generate a printable, human readable text file containing the encryption key.
+///
+/// This also includes a scanable QR code for fast key restore.
+fn paper_key(
+    path: Option<String>,
+    subject: Option<String>,
+    output_format: Option<PaperkeyFormat>,
+) -> Result<(), Error> {
+    let path = match path {
+        Some(path) => PathBuf::from(path),
+        None => {
+            let path = find_default_encryption_key()?
+                .ok_or_else(|| {
+                    format_err!("no encryption file provided and no default file found")
+                })?;
+            path
+        }
+    };
+
+    let data = file_get_contents(&path)?;
+    let data = std::str::from_utf8(&data)?;
+
+    let format = output_format.unwrap_or(PaperkeyFormat::Html);
+
+    match format {
+        PaperkeyFormat::Html => paperkey_html(data, subject),
+        PaperkeyFormat::Text => paperkey_text(data, subject),
+    }
+}
+
 pub fn cli() -> CliCommandMap {
     let key_create_cmd_def = CliCommand::new(&API_METHOD_CREATE)
         .arg_param(&["path"])
@@ -276,9 +337,214 @@ pub fn cli() -> CliCommandMap {
         .arg_param(&["path"])
         .completion_cb("path", tools::complete_file_name);
 
+    let paper_key_cmd_def = CliCommand::new(&API_METHOD_PAPER_KEY)
+        .arg_param(&["path"])
+        .completion_cb("path", tools::complete_file_name);
+
     CliCommandMap::new()
         .insert("create", key_create_cmd_def)
         .insert("create-master-key", key_create_master_key_cmd_def)
         .insert("import-master-pubkey", key_import_master_pubkey_cmd_def)
         .insert("change-passphrase", key_change_passphrase_cmd_def)
+        .insert("paper-key", paper_key_cmd_def)
+}
+
+fn paperkey_html(data: &str, subject: Option<String>) -> Result<(), Error> {
+
+    let img_size_pt = 500;
+
+    println!("<!DOCTYPE html>");
+    println!("<html lang=\"en\">");
+    println!("<head>");
+    println!("<meta charset=\"utf-8\">");
+    println!("<meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">");
+    println!("<title>Proxmox Backup Paperkey</title>");
+    println!("<style type=\"text/css\">");
+
+    println!("  p {{");
+    println!("    font-size: 12pt;");
+    println!("    font-family: monospace;");
+    println!("    white-space: pre-wrap;");
+    println!("    line-break: anywhere;");
+    println!("  }}");
+
+    println!("</style>");
+
+    println!("</head>");
+
+    println!("<body>");
+
+    if let Some(subject) = subject {
+        println!("<p>Subject: {}</p>", subject);
+    }
+
+    if data.starts_with("-----BEGIN ENCRYPTED PRIVATE KEY-----\n") {
+        let lines: Vec<String> = data.lines()
+            .map(|s| s.trim_end())
+            .filter(|s| !s.is_empty())
+            .map(String::from)
+            .collect();
+
+        if !lines[lines.len()-1].starts_with("-----END ENCRYPTED PRIVATE KEY-----") {
+            bail!("unexpected key format");
+        }
+
+        if lines.len() < 20 {
+            bail!("unexpected key format");
+        }
+
+        const BLOCK_SIZE: usize = 20;
+        let blocks = (lines.len() + BLOCK_SIZE -1)/BLOCK_SIZE;
+
+        for i in 0..blocks {
+            let start = i*BLOCK_SIZE;
+            let mut end = start + BLOCK_SIZE;
+            if end > lines.len() {
+                end = lines.len();
+            }
+            let data = &lines[start..end];
+
+            println!("<div style=\"page-break-inside: avoid;page-break-after: always\">");
+            println!("<p>");
+
+            for l in start..end {
+                println!("{:02}: {}", l, lines[l]);
+            }
+
+            println!("</p>");
+
+            let data = data.join("\n");
+            let qr_code = generate_qr_code("png", data.as_bytes())?;
+            let qr_code = base64::encode_config(&qr_code, base64::STANDARD_NO_PAD);
+
+            println!("<center>");
+            println!("<img");
+            println!("width=\"{}pt\" height=\"{}pt\"", img_size_pt, img_size_pt);
+            println!("src=\"data:image/png;base64,{}\"/>", qr_code);
+            println!("</center>");
+            println!("</div>");
+       }
+
+        println!("</body>");
+        println!("</html>");
+        return Ok(());
+    }
+
+    let key_config: KeyConfig = serde_json::from_str(&data)?;
+    let key_text = serde_json::to_string_pretty(&key_config)?;
+
+    println!("<div style=\"page-break-inside: avoid\">");
+
+    println!("<p>");
+
+    println!("-----BEGIN PROXMOX BACKUP KEY-----");
+
+    for line in key_text.lines() {
+        println!("{}", line);
+    }
+
+    println!("-----END PROXMOX BACKUP KEY-----");
+
+    println!("</p>");
+
+    let qr_code = generate_qr_code("png", key_text.as_bytes())?;
+    let qr_code = base64::encode_config(&qr_code, base64::STANDARD_NO_PAD);
+
+    println!("<center>");
+    println!("<img");
+    println!("width=\"{}pt\" height=\"{}pt\"", img_size_pt, img_size_pt);
+    println!("src=\"data:image/png;base64,{}\"/>", qr_code);
+    println!("</center>");
+
+    println!("</div>");
+
+    println!("</body>");
+    println!("</html>");
+
+    Ok(())
+}
+
+fn paperkey_text(data: &str, subject: Option<String>) -> Result<(), Error> {
+
+    if let Some(subject) = subject {
+        println!("Subject: {}\n", subject);
+    }
+
+    if data.starts_with("-----BEGIN ENCRYPTED PRIVATE KEY-----\n") {
+        let lines: Vec<String> = data.lines()
+            .map(|s| s.trim_end())
+            .filter(|s| !s.is_empty())
+            .map(String::from)
+            .collect();
+
+        if !lines[lines.len()-1].starts_with("-----END ENCRYPTED PRIVATE KEY-----") {
+            bail!("unexpected key format");
+        }
+
+        if lines.len() < 20 {
+            bail!("unexpected key format");
+        }
+
+        const BLOCK_SIZE: usize = 5;
+        let blocks = (lines.len() + BLOCK_SIZE -1)/BLOCK_SIZE;
+
+        for i in 0..blocks {
+            let start = i*BLOCK_SIZE;
+            let mut end = start + BLOCK_SIZE;
+            if end > lines.len() {
+                end = lines.len();
+            }
+            let data = &lines[start..end];
+
+            for l in start..end {
+                println!("{:-2}: {}", l, lines[l]);
+            }
+            let data = data.join("\n");
+            let qr_code = generate_qr_code("utf8i", data.as_bytes())?;
+            let qr_code = String::from_utf8(qr_code)
+                .map_err(|_| format_err!("Failed to read qr code (got non-utf8 data)"))?;
+            println!("{}", qr_code);
+            println!("{}", char::from(12u8)); // page break
+
+        }
+        return Ok(());
+    }
+
+    let key_config: KeyConfig = serde_json::from_str(&data)?;
+    let key_text = serde_json::to_string_pretty(&key_config)?;
+
+    println!("-----BEGIN PROXMOX BACKUP KEY-----");
+    println!("{}", key_text);
+    println!("-----END PROXMOX BACKUP KEY-----");
+
+    let qr_code = generate_qr_code("utf8i", key_text.as_bytes())?;
+    let qr_code = String::from_utf8(qr_code)
+        .map_err(|_| format_err!("Failed to read qr code (got non-utf8 data)"))?;
+
+    println!("{}", qr_code);
+
+    Ok(())
+}
+
+fn generate_qr_code(output_type: &str, data: &[u8]) -> Result<Vec<u8>, Error> {
+
+    let mut child = Command::new("qrencode")
+        .args(&["-t", output_type, "-m0", "-s1", "-lm", "--output", "-"])
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .spawn()?;
+
+    {
+        let stdin = child.stdin.as_mut()
+            .ok_or_else(|| format_err!("Failed to open stdin"))?;
+        stdin.write_all(data)
+            .map_err(|_| format_err!("Failed to write to stdin"))?;
+    }
+
+    let output = child.wait_with_output()
+        .map_err(|_| format_err!("Failed to read stdout"))?;
+
+    let output = crate::tools::command_output(output, None)?;
+
+    Ok(output)
 }

src/bin/proxmox_backup_client/mount.rs

@@ -101,7 +101,7 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let archive_name = tools::required_string_param(&param, "archive-name")?;
     let target = tools::required_string_param(&param, "target")?;
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
 
     record_repository(&repo);
 
@@ -141,7 +141,7 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
 
     let (manifest, _) = client.download_manifest().await?;
 
-    let file_info = manifest.lookup_file_info(&archive_name)?;
+    let file_info = manifest.lookup_file_info(&server_archive_name)?;
 
     if server_archive_name.ends_with(".didx") {
         let index = client.download_dynamic_index(&manifest, &server_archive_name).await?;

src/bin/proxmox_backup_client/task.rs

@@ -48,7 +48,7 @@ async fn task_list(param: Value) -> Result<Value, Error> {
     let output_format = get_output_format(&param);
 
     let repo = extract_repository_from_value(&param)?;
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
 
     let limit = param["limit"].as_u64().unwrap_or(50) as usize;
     let running = !param["all"].as_bool().unwrap_or(false);
@@ -96,7 +96,7 @@ async fn task_log(param: Value) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let upid =  tools::required_string_param(&param, "upid")?;
 
-    let client = connect(repo.host(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.user())?;
 
     display_task_log(client, upid, true).await?;
 
@@ -122,7 +122,7 @@ async fn task_stop(param: Value) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let upid_str =  tools::required_string_param(&param, "upid")?;
 
-    let mut client = connect(repo.host(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.user())?;
 
     let path = format!("api2/json/nodes/localhost/tasks/{}", upid_str);
     let _ = client.delete(&path, None).await?;

src/client/backup_reader.rs

@@ -1,16 +1,18 @@
 use anyhow::{format_err, Error};
-use std::io::{Read, Write, Seek, SeekFrom};
+use std::io::{Write, Seek, SeekFrom};
 use std::fs::File;
 use std::sync::Arc;
 use std::os::unix::fs::OpenOptionsExt;
 
-use chrono::{DateTime, Utc};
 use futures::future::AbortHandle;
 use serde_json::{json, Value};
 
 use proxmox::tools::digest_to_hex;
 
-use crate::backup::*;
+use crate::{
+    tools::compute_file_csum,
+    backup::*,
+};
 
 use super::{HttpClient, H2Client};
 
@@ -41,18 +43,18 @@ impl BackupReader {
         datastore: &str,
         backup_type: &str,
         backup_id: &str,
-        backup_time: DateTime<Utc>,
+        backup_time: i64,
         debug: bool,
     ) -> Result<Arc<BackupReader>, Error> {
 
         let param = json!({
             "backup-type": backup_type,
             "backup-id": backup_id,
-            "backup-time": backup_time.timestamp(),
+            "backup-time": backup_time,
             "store": datastore,
             "debug": debug,
         });
-        let req = HttpClient::request_builder(client.server(), "GET", "/api2/json/reader", Some(param)).unwrap();
+        let req = HttpClient::request_builder(client.server(), client.port(), "GET", "/api2/json/reader", Some(param)).unwrap();
 
         let (h2, abort) = client.start_h2_connection(req, String::from(PROXMOX_BACKUP_READER_PROTOCOL_ID_V1!())).await?;
 
@@ -220,29 +222,3 @@ impl BackupReader {
         Ok(index)
     }
 }
-
-pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
-
-    file.seek(SeekFrom::Start(0))?;
-
-    let mut hasher = openssl::sha::Sha256::new();
-    let mut buffer = proxmox::tools::vec::undefined(256*1024);
-    let mut size: u64 = 0;
-
-    loop {
-        let count = match file.read(&mut buffer) {
-            Ok(count) => count,
-            Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => { continue; }
-            Err(err) => return Err(err.into()),
-        };
-        if count == 0 {
-            break;
-        }
-        size += count as u64;
-        hasher.update(&buffer[..count]);
-    }
-
-    let csum = hasher.finish();
-
-    Ok((csum, size))
-}

src/client/backup_repo.rs

@@ -19,14 +19,22 @@ pub struct BackupRepository {
     user: Option<Userid>,
     /// The host name or IP address
     host: Option<String>,
+    /// The port
+    port: Option<u16>,
     /// The name of the datastore
     store: String,
 }
 
 impl BackupRepository {
 
-    pub fn new(user: Option<Userid>, host: Option<String>, store: String) -> Self {
-        Self { user, host, store }
+    pub fn new(user: Option<Userid>, host: Option<String>, port: Option<u16>, store: String) -> Self {
+        let host = match host {
+            Some(host) if (IP_V6_REGEX.regex_obj)().is_match(&host) => {
+                Some(format!("[{}]", host))
+            },
+            other => other,
+        };
+        Self { user, host, port, store }
     }
 
     pub fn user(&self) -> &Userid {
@@ -43,6 +51,13 @@ impl BackupRepository {
         "localhost"
     }
 
+    pub fn port(&self) -> u16 {
+        if let Some(port) = self.port {
+            return port;
+        }
+        8007
+    }
+
     pub fn store(&self) -> &str {
         &self.store
     }
@@ -50,13 +65,12 @@ impl BackupRepository {
 
 impl fmt::Display for BackupRepository {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-       if let Some(ref user) = self.user {
-           write!(f, "{}@{}:{}", user, self.host(), self.store)
-       } else if let Some(ref host) = self.host {
-           write!(f, "{}:{}", host, self.store)
-       } else {
-           write!(f, "{}", self.store)
-       }
+        match (&self.user, &self.host, self.port) {
+            (Some(user), _, _) => write!(f, "{}@{}:{}:{}", user, self.host(), self.port(), self.store),
+            (None, Some(host), None) => write!(f, "{}:{}", host, self.store),
+            (None, _, Some(port)) => write!(f, "{}:{}:{}", self.host(), port, self.store),
+            (None, None, None) => write!(f, "{}", self.store),
+        }
     }
 }
 
@@ -76,7 +90,8 @@ impl std::str::FromStr for BackupRepository {
         Ok(Self {
             user: cap.get(1).map(|m| Userid::try_from(m.as_str().to_owned())).transpose()?,
             host: cap.get(2).map(|m| m.as_str().to_owned()),
-            store: cap[3].to_owned(),
+            port: cap.get(3).map(|m| m.as_str().parse::<u16>()).transpose()?,
+            store: cap[4].to_owned(),
         })
     }
 }

src/client/backup_writer.rs

@@ -4,7 +4,6 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
 
 use anyhow::{bail, format_err, Error};
-use chrono::{DateTime, Utc};
 use futures::*;
 use futures::stream::Stream;
 use futures::future::AbortHandle;
@@ -51,20 +50,22 @@ impl BackupWriter {
         datastore: &str,
         backup_type: &str,
         backup_id: &str,
-        backup_time: DateTime<Utc>,
+        backup_time: i64,
         debug: bool,
+        benchmark: bool
     ) -> Result<Arc<BackupWriter>, Error> {
 
         let param = json!({
             "backup-type": backup_type,
             "backup-id": backup_id,
-            "backup-time": backup_time.timestamp(),
+            "backup-time": backup_time,
             "store": datastore,
-            "debug": debug
+            "debug": debug,
+            "benchmark": benchmark
         });
 
         let req = HttpClient::request_builder(
-            client.server(), "GET", "/api2/json/backup", Some(param)).unwrap();
+            client.server(), client.port(), "GET", "/api2/json/backup", Some(param)).unwrap();
 
         let (h2, abort) = client.start_h2_connection(req, String::from(PROXMOX_BACKUP_PROTOCOL_ID_V1!())).await?;
 

src/client/http_client.rs

@@ -1,8 +1,8 @@
 use std::io::Write;
 use std::task::{Context, Poll};
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, RwLock};
+use std::time::Duration;
 
-use chrono::Utc;
 use anyhow::{bail, format_err, Error};
 use futures::*;
 use http::Uri;
@@ -30,7 +30,7 @@ use crate::tools::{self, BroadcastFuture, DEFAULT_ENCODE_SET};
 
 #[derive(Clone)]
 pub struct AuthInfo {
-    pub username: String,
+    pub userid: Userid,
     pub ticket: String,
     pub token: String,
 }
@@ -99,8 +99,11 @@ impl HttpClientOptions {
 pub struct HttpClient {
     client: Client<HttpsConnector>,
     server: String,
+    port: u16,
     fingerprint: Arc<Mutex<Option<String>>>,
-    auth: BroadcastFuture<AuthInfo>,
+    first_auth: BroadcastFuture<()>,
+    auth: Arc<RwLock<AuthInfo>>,
+    ticket_abort: futures::future::AbortHandle,
     _options: HttpClientOptions,
 }
 
@@ -199,7 +202,7 @@ fn store_ticket_info(prefix: &str, server: &str, username: &str, ticket: &str, t
 
     let mut data = file_get_json(&path, Some(json!({})))?;
 
-    let now = Utc::now().timestamp();
+    let now = proxmox::tools::time::epoch_i64();
 
     data[server][username] = json!({ "timestamp": now, "ticket": ticket, "token": token});
 
@@ -230,7 +233,7 @@ fn load_ticket_info(prefix: &str, server: &str, userid: &Userid) -> Option<(Stri
     // usually /run/user/<uid>/...
     let path = base.place_runtime_file("tickets").ok()?;
     let data = file_get_json(&path, None).ok()?;
-    let now = Utc::now().timestamp();
+    let now = proxmox::tools::time::epoch_i64();
     let ticket_lifetime = tools::ticket::TICKET_LIFETIME - 60;
     let uinfo = data[server][userid.as_str()].as_object()?;
     let timestamp = uinfo["timestamp"].as_i64()?;
@@ -248,6 +251,7 @@ fn load_ticket_info(prefix: &str, server: &str, userid: &Userid) -> Option<(Stri
 impl HttpClient {
     pub fn new(
         server: &str,
+        port: u16,
         userid: &Userid,
         mut options: HttpClientOptions,
     ) -> Result<Self, Error> {
@@ -292,7 +296,6 @@ impl HttpClient {
 
         let mut httpc = hyper::client::HttpConnector::new();
         httpc.set_nodelay(true); // important for h2 download performance!
-        httpc.set_recv_buffer_size(Some(1024*1024)); //important for h2 download performance!
         httpc.enforce_http(false); // we want https...
 
         let https = HttpsConnector::with_connector(httpc, ssl_connector_builder.build());
@@ -319,29 +322,69 @@ impl HttpClient {
             }
         };
 
+        let auth = Arc::new(RwLock::new(AuthInfo {
+            userid: userid.clone(),
+            ticket: password.clone(),
+            token: "".to_string(),
+        }));
+
+        let server2 = server.to_string();
+        let client2 = client.clone();
+        let auth2 = auth.clone();
+        let prefix2 = options.prefix.clone();
+
+        let renewal_future = async move {
+            loop {
+                tokio::time::delay_for(Duration::new(60*15,  0)).await; // 15 minutes
+                let (userid, ticket) = {
+                    let authinfo = auth2.read().unwrap().clone();
+                    (authinfo.userid, authinfo.ticket)
+                };
+                match Self::credentials(client2.clone(), server2.clone(), port, userid, ticket).await {
+                    Ok(auth) => {
+                        if use_ticket_cache & &prefix2.is_some() {
+                            let _ = store_ticket_info(prefix2.as_ref().unwrap(), &server2, &auth.userid.to_string(), &auth.ticket, &auth.token);
+                        }
+                        *auth2.write().unwrap() = auth;
+                    },
+                    Err(err) => {
+                        eprintln!("re-authentication failed: {}", err);
+                        return;
+                    }
+                }
+            }
+        };
+
+        let (renewal_future, ticket_abort) = futures::future::abortable(renewal_future);
+
         let login_future = Self::credentials(
             client.clone(),
             server.to_owned(),
+            port,
             userid.to_owned(),
             password.to_owned(),
         ).map_ok({
             let server = server.to_string();
             let prefix = options.prefix.clone();
+            let authinfo = auth.clone();
 
             move |auth| {
                 if use_ticket_cache & &prefix.is_some() {
-                    let _ = store_ticket_info(prefix.as_ref().unwrap(), &server, &auth.username, &auth.ticket, &auth.token);
+                    let _ = store_ticket_info(prefix.as_ref().unwrap(), &server, &auth.userid.to_string(), &auth.ticket, &auth.token);
                 }
-
-                auth
+                *authinfo.write().unwrap() = auth;
+                tokio::spawn(renewal_future);
             }
         });
 
         Ok(Self {
             client,
             server: String::from(server),
+            port,
             fingerprint: verified_fingerprint,
-            auth: BroadcastFuture::new(Box::new(login_future)),
+            auth,
+            ticket_abort,
+            first_auth: BroadcastFuture::new(Box::new(login_future)),
             _options: options,
         })
     }
@@ -351,7 +394,9 @@ impl HttpClient {
     /// Login is done on demand, so this is only required if you need
     /// access to authentication data in 'AuthInfo'.
     pub async fn login(&self) -> Result<AuthInfo, Error> {
-        self.auth.listen().await
+        self.first_auth.listen().await?;
+        let authinfo = self.auth.read().unwrap();
+        Ok(authinfo.clone())
     }
 
     /// Returns the optional fingerprint passed to the new() constructor.
@@ -445,7 +490,7 @@ impl HttpClient {
         path: &str,
         data: Option<Value>,
     ) -> Result<Value, Error> {
-        let req = Self::request_builder(&self.server, "GET", path, data).unwrap();
+        let req = Self::request_builder(&self.server, self.port, "GET", path, data)?;
         self.request(req).await
     }
 
@@ -454,7 +499,7 @@ impl HttpClient {
         path: &str,
         data: Option<Value>,
     ) -> Result<Value, Error> {
-        let req = Self::request_builder(&self.server, "DELETE", path, data).unwrap();
+        let req = Self::request_builder(&self.server, self.port, "DELETE", path, data)?;
         self.request(req).await
     }
 
@@ -463,7 +508,7 @@ impl HttpClient {
         path: &str,
         data: Option<Value>,
     ) -> Result<Value, Error> {
-        let req = Self::request_builder(&self.server, "POST", path, data).unwrap();
+        let req = Self::request_builder(&self.server, self.port, "POST", path, data)?;
         self.request(req).await
     }
 
@@ -472,7 +517,7 @@ impl HttpClient {
         path: &str,
         output: &mut (dyn Write + Send),
     ) -> Result<(), Error> {
-        let mut req = Self::request_builder(&self.server, "GET", path, None).unwrap();
+        let mut req = Self::request_builder(&self.server, self.port, "GET", path, None)?;
 
         let client = self.client.clone();
 
@@ -508,7 +553,7 @@ impl HttpClient {
     ) -> Result<Value, Error> {
 
         let path = path.trim_matches('/');
-        let mut url = format!("https://{}:8007/{}", &self.server, path);
+        let mut url = format!("https://{}:{}/{}", &self.server, self.port, path);
 
         if let Some(data) = data {
             let query = tools::json_object_to_query(data).unwrap();
@@ -583,14 +628,15 @@ impl HttpClient {
     async fn credentials(
         client: Client<HttpsConnector>,
         server: String,
+        port: u16,
         username: Userid,
         password: String,
     ) -> Result<AuthInfo, Error> {
         let data = json!({ "username": username, "password": password });
-        let req = Self::request_builder(&server, "POST", "/api2/json/access/ticket", Some(data)).unwrap();
+        let req = Self::request_builder(&server, port, "POST", "/api2/json/access/ticket", Some(data))?;
         let cred = Self::api_request(client, req).await?;
         let auth = AuthInfo {
-            username: cred["data"]["username"].as_str().unwrap().to_owned(),
+            userid: cred["data"]["username"].as_str().unwrap().parse()?,
             ticket: cred["data"]["ticket"].as_str().unwrap().to_owned(),
             token: cred["data"]["CSRFPreventionToken"].as_str().unwrap().to_owned(),
         };
@@ -631,9 +677,13 @@ impl HttpClient {
         &self.server
     }
 
-    pub fn request_builder(server: &str, method: &str, path: &str, data: Option<Value>) -> Result<Request<Body>, Error> {
+    pub fn port(&self) -> u16 {
+        self.port
+    }
+
+    pub fn request_builder(server: &str, port: u16, method: &str, path: &str, data: Option<Value>) -> Result<Request<Body>, Error> {
         let path = path.trim_matches('/');
-        let url: Uri = format!("https://{}:8007/{}", server, path).parse()?;
+        let url: Uri = format!("https://{}:{}/{}", server, port, path).parse()?;
 
         if let Some(data) = data {
             if method == "POST" {
@@ -646,7 +696,7 @@ impl HttpClient {
                 return Ok(request);
             } else {
                 let query = tools::json_object_to_query(data)?;
-                let url: Uri = format!("https://{}:8007/{}?{}", server, path, query).parse()?;
+                let url: Uri = format!("https://{}:{}/{}?{}", server, port, path, query).parse()?;
                 let request = Request::builder()
                     .method(method)
                     .uri(url)
@@ -668,6 +718,12 @@ impl HttpClient {
     }
 }
 
+impl Drop for HttpClient {
+    fn drop(&mut self) {
+        self.ticket_abort.abort();
+    }
+}
+
 
 #[derive(Clone)]
 pub struct H2Client {

src/client/pull.rs

@@ -3,15 +3,20 @@
 use anyhow::{bail, format_err, Error};
 use serde_json::json;
 use std::convert::TryFrom;
-use std::sync::Arc;
-use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::collections::{HashSet, HashMap};
 use std::io::{Seek, SeekFrom};
+use std::time::SystemTime;
+use std::sync::atomic::{AtomicUsize, Ordering};
 
 use proxmox::api::error::{StatusCode, HttpError};
-use crate::server::{WorkerTask};
-use crate::backup::*;
-use crate::api2::types::*;
-use super::*;
+use crate::{
+    tools::{ParallelHandler, compute_file_csum},
+    server::WorkerTask,
+    backup::*,
+    api2::types::*,
+    client::*,
+};
 
 
 // fixme: implement filters
@@ -19,27 +24,86 @@ use super::*;
 // Todo: correctly lock backup groups
 
 async fn pull_index_chunks<I: IndexFile>(
-    _worker: &WorkerTask,
-    chunk_reader: &mut RemoteChunkReader,
+    worker: &WorkerTask,
+    chunk_reader: RemoteChunkReader,
     target: Arc<DataStore>,
     index: I,
+    downloaded_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
 ) -> Result<(), Error> {
 
+    use futures::stream::{self, StreamExt, TryStreamExt};
 
-    for pos in 0..index.index_count() {
-        let info = index.chunk_info(pos).unwrap();
-        let chunk_exists = target.cond_touch_chunk(&info.digest, false)?;
-        if chunk_exists {
-            //worker.log(format!("chunk {} exists {}", pos, proxmox::tools::digest_to_hex(digest)));
-            continue;
-        }
-        //worker.log(format!("sync {} chunk {}", pos, proxmox::tools::digest_to_hex(digest)));
-        let chunk = chunk_reader.read_raw_chunk(&info.digest).await?;
+    let start_time = SystemTime::now();
 
-        chunk.verify_unencrypted(info.size() as usize, &info.digest)?;
+    let stream = stream::iter(
+        (0..index.index_count())
+            .map(|pos| index.chunk_info(pos).unwrap())
+            .filter(|info| {
+                let mut guard = downloaded_chunks.lock().unwrap();
+                let done = guard.contains(&info.digest);
+                if !done {
+                    // Note: We mark a chunk as downloaded before its actually downloaded
+                    // to avoid duplicate downloads.
+                    guard.insert(info.digest);
+                }
+                !done
+            })
+    );
 
-        target.insert_chunk(&chunk, &info.digest)?;
-    }
+    let target2 = target.clone();
+    let verify_pool = ParallelHandler::new(
+        "sync chunk writer", 4,
+        move |(chunk, digest, size): (DataBlob, [u8;32], u64)|  {
+            // println!("verify and write {}", proxmox::tools::digest_to_hex(&digest));
+            chunk.verify_unencrypted(size as usize, &digest)?;
+            target2.insert_chunk(&chunk, &digest)?;
+            Ok(())
+       }
+    );
+
+    let verify_and_write_channel = verify_pool.channel();
+
+    let bytes = Arc::new(AtomicUsize::new(0));
+
+    stream
+        .map(|info| {
+
+            let target = Arc::clone(&target);
+            let chunk_reader = chunk_reader.clone();
+            let bytes = Arc::clone(&bytes);
+            let verify_and_write_channel = verify_and_write_channel.clone();
+
+            Ok::<_, Error>(async move {
+                let chunk_exists = crate::tools::runtime::block_in_place(|| target.cond_touch_chunk(&info.digest, false))?;
+                if chunk_exists {
+                    //worker.log(format!("chunk {} exists {}", pos, proxmox::tools::digest_to_hex(digest)));
+                    return Ok::<_, Error>(());
+                }
+                //worker.log(format!("sync {} chunk {}", pos, proxmox::tools::digest_to_hex(digest)));
+                let chunk = chunk_reader.read_raw_chunk(&info.digest).await?;
+                let raw_size = chunk.raw_size() as usize;
+
+                // decode, verify and write in a separate threads to maximize throughput
+                crate::tools::runtime::block_in_place(|| verify_and_write_channel.send((chunk, info.digest, info.size())))?;
+
+                bytes.fetch_add(raw_size, Ordering::SeqCst);
+
+                Ok(())
+           })
+        })
+        .try_buffer_unordered(20)
+        .try_for_each(|_res| futures::future::ok(()))
+        .await?;
+
+    drop(verify_and_write_channel);
+
+    verify_pool.complete()?;
+
+    let elapsed = start_time.elapsed()?.as_secs_f64();
+
+    let bytes = bytes.load(Ordering::SeqCst);
+
+    worker.log(format!("downloaded {} bytes ({} MiB/s)", bytes, (bytes as f64)/(1024.0*1024.0*elapsed)));
 
     Ok(())
 }
@@ -52,6 +116,7 @@ async fn download_manifest(
     let mut tmp_manifest_file = std::fs::OpenOptions::new()
         .write(true)
         .create(true)
+        .truncate(true)
         .read(true)
         .open(&filename)?;
 
@@ -85,6 +150,7 @@ async fn pull_single_archive(
     tgt_store: Arc<DataStore>,
     snapshot: &BackupDir,
     archive_info: &FileInfo,
+    downloaded_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
 ) -> Result<(), Error> {
 
     let archive_name = &archive_info.filename;
@@ -111,7 +177,7 @@ async fn pull_single_archive(
             let (csum, size) = index.compute_csum();
             verify_archive(archive_info, &csum, size)?;
 
-            pull_index_chunks(worker, chunk_reader, tgt_store.clone(), index).await?;
+            pull_index_chunks(worker, chunk_reader.clone(), tgt_store.clone(), index, downloaded_chunks).await?;
         }
         ArchiveType::FixedIndex => {
             let index = FixedIndexReader::new(tmpfile)
@@ -119,7 +185,7 @@ async fn pull_single_archive(
             let (csum, size) = index.compute_csum();
             verify_archive(archive_info, &csum, size)?;
 
-            pull_index_chunks(worker, chunk_reader, tgt_store.clone(), index).await?;
+            pull_index_chunks(worker, chunk_reader.clone(), tgt_store.clone(), index, downloaded_chunks).await?;
         }
         ArchiveType::Blob => {
             let (csum, size) = compute_file_csum(&mut tmpfile)?;
@@ -165,6 +231,7 @@ async fn pull_snapshot(
     reader: Arc<BackupReader>,
     tgt_store: Arc<DataStore>,
     snapshot: &BackupDir,
+    downloaded_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
 ) -> Result<(), Error> {
 
     let mut manifest_name = tgt_store.base_path();
@@ -218,6 +285,7 @@ async fn pull_snapshot(
                 try_client_log_download(worker, reader, &client_log_name).await?;
             }
             worker.log("no data changes");
+            let _ = std::fs::remove_file(&tmp_manifest_name);
             return Ok(()); // nothing changed
         }
     }
@@ -273,6 +341,7 @@ async fn pull_snapshot(
             tgt_store.clone(),
             snapshot,
             &item,
+            downloaded_chunks.clone(),
         ).await?;
     }
 
@@ -295,6 +364,7 @@ pub async fn pull_snapshot_from(
     reader: Arc<BackupReader>,
     tgt_store: Arc<DataStore>,
     snapshot: &BackupDir,
+    downloaded_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
 ) -> Result<(), Error> {
 
     let (_path, is_new, _snap_lock) = tgt_store.create_locked_backup_dir(&snapshot)?;
@@ -302,7 +372,7 @@ pub async fn pull_snapshot_from(
     if is_new {
         worker.log(format!("sync snapshot {:?}", snapshot.relative_path()));
 
-        if let Err(err) = pull_snapshot(worker, reader, tgt_store.clone(), &snapshot).await {
+        if let Err(err) = pull_snapshot(worker, reader, tgt_store.clone(), &snapshot, downloaded_chunks).await {
             if let Err(cleanup_err) = tgt_store.remove_backup_dir(&snapshot, true) {
                 worker.log(format!("cleanup error - {}", cleanup_err));
             }
@@ -311,7 +381,7 @@ pub async fn pull_snapshot_from(
         worker.log(format!("sync snapshot {:?} done", snapshot.relative_path()));
     } else {
         worker.log(format!("re-sync snapshot {:?}", snapshot.relative_path()));
-        pull_snapshot(worker, reader, tgt_store.clone(), &snapshot).await?;
+        pull_snapshot(worker, reader, tgt_store.clone(), &snapshot, downloaded_chunks).await?;
         worker.log(format!("re-sync snapshot {:?} done", snapshot.relative_path()));
     }
 
@@ -325,6 +395,7 @@ pub async fn pull_group(
     tgt_store: Arc<DataStore>,
     group: &BackupGroup,
     delete: bool,
+    progress: Option<(usize, usize)>, // (groups_done, group_count)
 ) -> Result<(), Error> {
 
     let path = format!("api2/json/admin/datastore/{}/snapshots", src_repo.store());
@@ -346,8 +417,21 @@ pub async fn pull_group(
 
     let mut remote_snapshots = std::collections::HashSet::new();
 
-    for item in list {
-        let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time);
+    let (per_start, per_group) = if let Some((groups_done, group_count)) = progress {
+        let per_start = (groups_done as f64)/(group_count as f64);
+        let per_group = 1.0/(group_count as f64);
+        (per_start, per_group)
+    } else {
+        (0.0, 1.0)
+    };
+
+    // start with 16384 chunks (up to 65GB)
+    let downloaded_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*64)));
+
+    let snapshot_count = list.len();
+
+    for (pos, item) in list.into_iter().enumerate() {
+        let snapshot = BackupDir::new(item.backup_type, item.backup_id, item.backup_time)?;
 
         // in-progress backups can't be synced
         if let None = item.size {
@@ -367,7 +451,7 @@ pub async fn pull_group(
             .password(Some(auth_info.ticket.clone()))
             .fingerprint(fingerprint.clone());
 
-        let new_client = HttpClient::new(src_repo.host(), src_repo.user(), options)?;
+        let new_client = HttpClient::new(src_repo.host(), src_repo.port(), src_repo.user(), options)?;
 
         let reader = BackupReader::start(
             new_client,
@@ -379,7 +463,13 @@ pub async fn pull_group(
             true,
         ).await?;
 
-        pull_snapshot_from(worker, reader, tgt_store.clone(), &snapshot).await?;
+        let result = pull_snapshot_from(worker, reader, tgt_store.clone(), &snapshot, downloaded_chunks.clone()).await;
+
+        let percentage = (pos as f64)/(snapshot_count as f64);
+        let percentage = per_start + percentage*per_group;
+        worker.log(format!("percentage done: {:.2}%", percentage*100.0));
+
+        result?; // stop on error
     }
 
     if delete {
@@ -429,6 +519,9 @@ pub async fn pull_store(
         new_groups.insert(BackupGroup::new(&item.backup_type, &item.backup_id));
     }
 
+    let group_count = list.len();
+    let mut groups_done = 0;
+
     for item in list {
         let group = BackupGroup::new(&item.backup_type, &item.backup_id);
 
@@ -437,15 +530,24 @@ pub async fn pull_store(
         if userid != owner { // only the owner is allowed to create additional snapshots
             worker.log(format!("sync group {}/{} failed - owner check failed ({} != {})",
                                item.backup_type, item.backup_id, userid, owner));
-            errors = true;
-            continue; // do not stop here, instead continue
-        }
+            errors = true; // do not stop here, instead continue
 
-        if let Err(err) = pull_group(worker, client, src_repo, tgt_store.clone(), &group, delete).await {
-            worker.log(format!("sync group {}/{} failed - {}", item.backup_type, item.backup_id, err));
-            errors = true;
-            continue; // do not stop here, instead continue
+        } else {
+
+            if let Err(err) = pull_group(
+                worker,
+                client,
+                src_repo,
+                tgt_store.clone(),
+                &group,
+                delete,
+                Some((groups_done, group_count)),
+            ).await {
+                worker.log(format!("sync group {}/{} failed - {}", item.backup_type, item.backup_id, err));
+                errors = true; // do not stop here, instead continue
+            }
         }
+        groups_done += 1;
     }
 
     if delete {

src/client/remote_chunk_reader.rs

@@ -15,7 +15,7 @@ pub struct RemoteChunkReader {
     client: Arc<BackupReader>,
     crypt_config: Option<Arc<CryptConfig>>,
     crypt_mode: CryptMode,
-    cache_hint: HashMap<[u8; 32], usize>,
+    cache_hint: Arc<HashMap<[u8; 32], usize>>,
     cache: Arc<Mutex<HashMap<[u8; 32], Vec<u8>>>>,
 }
 
@@ -33,7 +33,7 @@ impl RemoteChunkReader {
             client,
             crypt_config,
             crypt_mode,
-            cache_hint,
+            cache_hint: Arc::new(cache_hint),
             cache: Arc::new(Mutex::new(HashMap::new())),
         }
     }

src/config/datastore.rs

@@ -44,6 +44,10 @@ pub const DIR_NAME_SCHEMA: Schema = StringSchema::new("Directory name").schema()
             optional: true,
             schema: PRUNE_SCHEDULE_SCHEMA,
         },
+        "verify-schedule": {
+            optional: true,
+            schema: VERIFY_SCHEDULE_SCHEMA,
+        },
         "keep-last": {
             optional: true,
             schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -83,6 +87,8 @@ pub struct DataStoreConfig {
     #[serde(skip_serializing_if="Option::is_none")]
     pub prune_schedule: Option<String>,
     #[serde(skip_serializing_if="Option::is_none")]
+    pub verify_schedule: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
     pub keep_last: Option<u64>,
     #[serde(skip_serializing_if="Option::is_none")]
     pub keep_hourly: Option<u64>,

src/config/jobstate.rs

@@ -48,7 +48,6 @@ use proxmox::tools::fs::{
 use serde::{Deserialize, Serialize};
 
 use crate::server::{upid_read_status, worker_is_active_local, TaskState, UPID};
-use crate::tools::epoch_now_u64;
 
 #[serde(rename_all = "kebab-case")]
 #[derive(Serialize, Deserialize)]
@@ -98,7 +97,7 @@ where
 {
     let mut path = path.as_ref().to_path_buf();
     path.set_extension("lck");
-    let lock = open_file_locked(&path, Duration::new(10, 0))?;
+    let lock = open_file_locked(&path, Duration::new(10, 0), true)?;
     let backup_user = crate::backup::backup_user()?;
     nix::unistd::chown(&path, Some(backup_user.uid), Some(backup_user.gid))?;
     Ok(lock)
@@ -178,7 +177,7 @@ impl JobState {
             }
         } else {
             Ok(JobState::Created {
-                time: epoch_now_u64()? as i64 - 30,
+                time: proxmox::tools::time::epoch_i64() - 30,
             })
         }
     }
@@ -199,7 +198,7 @@ impl Job {
             jobtype: jobtype.to_string(),
             jobname: jobname.to_string(),
             state: JobState::Created {
-                time: epoch_now_u64()? as i64,
+                time: proxmox::tools::time::epoch_i64(),
             },
             _lock,
         })

src/config/network.rs

@@ -17,7 +17,7 @@ pub use lexer::*;
 mod parser;
 pub use parser::*;
 
-use crate::api2::types::{Interface, NetworkConfigMethod, NetworkInterfaceType, LinuxBondMode};
+use crate::api2::types::{Interface, NetworkConfigMethod, NetworkInterfaceType, LinuxBondMode, BondXmitHashPolicy};
 
 lazy_static!{
     static ref PHYSICAL_NIC_REGEX: Regex = Regex::new(r"^(?:eth\d+|en[^:.]+|ib\d+)$").unwrap();
@@ -44,6 +44,19 @@ pub fn bond_mode_to_str(mode: LinuxBondMode) -> &'static str {
     }
 }
 
+pub fn bond_xmit_hash_policy_from_str(s: &str) -> Result<BondXmitHashPolicy, Error> {
+    BondXmitHashPolicy::deserialize(s.into_deserializer())
+        .map_err(|_: value::Error| format_err!("invalid bond_xmit_hash_policy '{}'", s))
+}
+
+pub fn bond_xmit_hash_policy_to_str(policy: &BondXmitHashPolicy) -> &'static str {
+    match policy {
+        BondXmitHashPolicy::layer2 => "layer2",
+        BondXmitHashPolicy::layer2_3 => "layer2+3",
+        BondXmitHashPolicy::layer3_4 => "layer3+4",
+    }
+}
+
 impl Interface {
 
     pub fn new(name: String) -> Self {
@@ -67,6 +80,8 @@ impl Interface {
             bridge_vlan_aware: None,
             slaves: None,
             bond_mode: None,
+            bond_primary: None,
+            bond_xmit_hash_policy: None,
         }
     }
 
@@ -169,6 +184,19 @@ impl Interface {
             NetworkInterfaceType::Bond => {
                 let mode = self.bond_mode.unwrap_or(LinuxBondMode::balance_rr);
                 writeln!(w, "\tbond-mode {}", bond_mode_to_str(mode))?;
+                if let Some(primary) = &self.bond_primary {
+                    if mode == LinuxBondMode::active_backup {
+                        writeln!(w, "\tbond-primary {}", primary)?;
+                    }
+                }
+
+                if let Some(xmit_policy) = &self.bond_xmit_hash_policy {
+                    if mode == LinuxBondMode::ieee802_3ad ||
+                       mode == LinuxBondMode::balance_xor
+                    {
+                        writeln!(w, "\tbond_xmit_hash_policy {}", bond_xmit_hash_policy_to_str(xmit_policy))?;
+                    }
+                }
 
                 let slaves = self.slaves.as_ref().unwrap_or(&EMPTY_LIST);
                 if slaves.is_empty() {

src/config/network/helper.rs

@@ -149,7 +149,7 @@ pub fn compute_file_diff(filename: &str, shadow: &str) -> Result<String, Error>
         .output()
         .map_err(|err| format_err!("failed to execute diff - {}", err))?;
 
-    let diff = crate::tools::command_output(output, Some(|c| c == 0 || c == 1))
+    let diff = crate::tools::command_output_as_string(output, Some(|c| c == 0 || c == 1))
         .map_err(|err| format_err!("diff failed: {}", err))?;
 
     Ok(diff)

src/config/network/lexer.rs

@@ -26,6 +26,8 @@ pub enum Token {
     BridgeVlanAware,
     BondSlaves,
     BondMode,
+    BondPrimary,
+    BondXmitHashPolicy,
     EOF,
 }
 
@@ -51,7 +53,10 @@ lazy_static! {
         map.insert("bond-slaves", Token::BondSlaves);
         map.insert("bond_slaves", Token::BondSlaves);
         map.insert("bond-mode", Token::BondMode);
-        map.insert("bond_mode", Token::BondMode);
+        map.insert("bond-primary", Token::BondPrimary);
+        map.insert("bond_primary", Token::BondPrimary);
+        map.insert("bond_xmit_hash_policy", Token::BondXmitHashPolicy);
+        map.insert("bond-xmit-hash-policy", Token::BondXmitHashPolicy);
         map
     };
 }

src/config/network/parser.rs

@@ -9,7 +9,7 @@ use regex::Regex;
 use super::helper::*;
 use super::lexer::*;
 
-use super::{NetworkConfig, NetworkOrderEntry, Interface, NetworkConfigMethod, NetworkInterfaceType, bond_mode_from_str};
+use super::{NetworkConfig, NetworkOrderEntry, Interface, NetworkConfigMethod, NetworkInterfaceType, bond_mode_from_str, bond_xmit_hash_policy_from_str};
 
 pub struct NetworkParser<R: BufRead> {
     input: Peekable<Lexer<R>>,
@@ -243,6 +243,18 @@ impl <R: BufRead> NetworkParser<R> {
                     interface.bond_mode = Some(bond_mode_from_str(&mode)?);
                     self.eat(Token::Newline)?;
                 }
+                Token::BondPrimary => {
+                    self.eat(Token::BondPrimary)?;
+                    let primary = self.next_text()?;
+                    interface.bond_primary = Some(primary);
+                    self.eat(Token::Newline)?;
+                }
+                Token::BondXmitHashPolicy => {
+                    self.eat(Token::BondXmitHashPolicy)?;
+                    let policy = bond_xmit_hash_policy_from_str(&self.next_text()?)?;
+                    interface.bond_xmit_hash_policy = Some(policy);
+                    self.eat(Token::Newline)?;
+                }
                 Token::Netmask => bail!("netmask is deprecated and no longer supported"),
 
                 _ => { // parse addon attributes

src/config/remote.rs

@@ -39,6 +39,11 @@ pub const REMOTE_PASSWORD_SCHEMA: Schema = StringSchema::new("Password or auth t
         host: {
             schema: DNS_NAME_OR_IP_SCHEMA,
         },
+        port: {
+            optional: true,
+            description: "The (optional) port",
+            type: u16,
+        },
         userid: {
             type: Userid,
         },
@@ -58,6 +63,8 @@ pub struct Remote {
     #[serde(skip_serializing_if="Option::is_none")]
     pub comment: Option<String>,
     pub host: String,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub port: Option<u16>,
     pub userid: Userid,
     #[serde(skip_serializing_if="String::is_empty")]
     #[serde(with = "proxmox::tools::serde::string_as_base64")]

src/pxar/catalog.rs

@@ -17,17 +17,3 @@ pub trait BackupCatalogWriter {
     fn add_fifo(&mut self, name: &CStr) -> Result<(), Error>;
     fn add_socket(&mut self, name: &CStr) -> Result<(), Error>;
 }
-
-pub struct DummyCatalogWriter();
-
-impl BackupCatalogWriter for DummyCatalogWriter {
-    fn start_directory(&mut self, _name: &CStr) -> Result<(), Error> { Ok(()) }
-    fn end_directory(&mut self) -> Result<(), Error> { Ok(()) }
-    fn add_file(&mut self, _name: &CStr, _size: u64, _mtime: u64) -> Result<(), Error> { Ok(()) }
-    fn add_symlink(&mut self, _name: &CStr) -> Result<(), Error> { Ok(()) }
-    fn add_hardlink(&mut self, _name: &CStr) -> Result<(), Error> { Ok(()) }
-    fn add_block_device(&mut self, _name: &CStr) -> Result<(), Error> { Ok(()) }
-    fn add_char_device(&mut self, _name: &CStr) -> Result<(), Error> { Ok(()) }
-    fn add_fifo(&mut self, _name: &CStr) -> Result<(), Error> { Ok(()) }
-    fn add_socket(&mut self, _name: &CStr) -> Result<(), Error> { Ok(()) }
-}

src/pxar/tools.rs

@@ -8,7 +8,7 @@ use std::path::Path;
 use anyhow::{bail, format_err, Error};
 use nix::sys::stat::Mode;
 
-use pxar::{mode, Entry, EntryKind, Metadata};
+use pxar::{mode, Entry, EntryKind, Metadata, format::StatxTimestamp};
 
 /// Get the file permissions as `nix::Mode`
 pub fn perms_from_metadata(meta: &Metadata) -> Result<Mode, Error> {
@@ -114,13 +114,17 @@ fn mode_string(entry: &Entry) -> String {
     )
 }
 
-pub fn format_single_line_entry(entry: &Entry) -> String {
-    use chrono::offset::TimeZone;
+fn format_mtime(mtime: &StatxTimestamp) -> String {
+    if let Ok(s) = proxmox::tools::time::strftime_local("%Y-%m-%d %H:%M:%S", mtime.secs) {
+        return s;
+    }
+    format!("{}.{}", mtime.secs, mtime.nanos)
+}
 
+pub fn format_single_line_entry(entry: &Entry) -> String {
     let mode_string = mode_string(entry);
 
     let meta = entry.metadata();
-    let mtime = chrono::Local.timestamp(meta.stat.mtime.secs, meta.stat.mtime.nanos);
 
     let (size, link) = match entry.kind() {
         EntryKind::File { size, .. } => (format!("{}", *size), String::new()),
@@ -134,7 +138,7 @@ pub fn format_single_line_entry(entry: &Entry) -> String {
         "{} {:<13} {} {:>8} {:?}{}",
         mode_string,
         format!("{}/{}", meta.stat.uid, meta.stat.gid),
-        mtime.format("%Y-%m-%d %H:%M:%S"),
+        format_mtime(&meta.stat.mtime),
         size,
         entry.path(),
         link,
@@ -142,12 +146,9 @@ pub fn format_single_line_entry(entry: &Entry) -> String {
 }
 
 pub fn format_multi_line_entry(entry: &Entry) -> String {
-    use chrono::offset::TimeZone;
-
     let mode_string = mode_string(entry);
 
     let meta = entry.metadata();
-    let mtime = chrono::Local.timestamp(meta.stat.mtime.secs, meta.stat.mtime.nanos);
 
     let (size, link, type_name) = match entry.kind() {
         EntryKind::File { size, .. } => (format!("{}", *size), String::new(), "file"),
@@ -196,6 +197,6 @@ pub fn format_multi_line_entry(entry: &Entry) -> String {
         mode_string,
         meta.stat.uid,
         meta.stat.gid,
-        mtime.format("%Y-%m-%d %H:%M:%S"),
+        format_mtime(&meta.stat.mtime),
     )
 }

src/rrd/cache.rs

@@ -8,7 +8,6 @@ use lazy_static::lazy_static;
 use proxmox::tools::fs::{create_path, CreateOptions};
 
 use crate::api2::types::{RRDMode, RRDTimeFrameResolution};
-use crate::tools::epoch_now_f64;
 
 use super::*;
 
@@ -42,7 +41,7 @@ pub fn update_value(rel_path: &str, value: f64, dst: DST, save: bool) -> Result<
     std::fs::create_dir_all(path.parent().unwrap())?;
 
     let mut map = RRD_CACHE.write().unwrap();
-    let now = epoch_now_f64()?;
+    let now = proxmox::tools::time::epoch_f64();
 
     if let Some(rrd) = map.get_mut(rel_path) {
         rrd.update(now, value);

src/rrd/rrd.rs

@@ -60,7 +60,7 @@ impl RRA {
 
         let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso;
         let min_time = (min_time/reso + 1)*reso;
-        let mut t = last_update - (RRD_DATA_ENTRIES as u64)*reso;
+        let mut t = last_update.saturating_sub((RRD_DATA_ENTRIES as u64)*reso);
         let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize;
         for _ in 0..RRD_DATA_ENTRIES {
             t += reso; index = (index + 1) % RRD_DATA_ENTRIES;

src/server/rest.rs

@@ -313,7 +313,13 @@ pub async fn handle_api_request<Env: RpcEnvironment, S: 'static + BuildHasher +
     Ok(resp)
 }
 
-fn get_index(userid: Option<Userid>, token: Option<String>, api: &Arc<ApiConfig>, parts: Parts) ->  Response<Body> {
+fn get_index(
+    userid: Option<Userid>,
+    token: Option<String>,
+    language: Option<String>,
+    api: &Arc<ApiConfig>,
+    parts: Parts,
+) ->  Response<Body> {
 
     let nodename = proxmox::tools::nodename();
     let userid = userid.as_ref().map(|u| u.as_str()).unwrap_or("");
@@ -333,10 +339,18 @@ fn get_index(userid: Option<Userid>, token: Option<String>, api: &Arc<ApiConfig>
         }
     }
 
+    let mut lang = String::from("");
+    if let Some(language) = language {
+        if Path::new(&format!("/usr/share/pbs-i18n/pbs-lang-{}.js", language)).exists() {
+            lang = language;
+        }
+    }
+
     let data = json!({
         "NodeName": nodename,
         "UserName": userid,
         "CSRFPreventionToken": token,
+        "language": lang,
         "debug": debug,
     });
 
@@ -441,12 +455,14 @@ async fn handle_static_file_download(filename: PathBuf) ->  Result<Response<Body
     }
 }
 
-fn extract_auth_data(headers: &http::HeaderMap) -> (Option<String>, Option<String>) {
+fn extract_auth_data(headers: &http::HeaderMap) -> (Option<String>, Option<String>, Option<String>) {
 
     let mut ticket = None;
+    let mut language = None;
     if let Some(raw_cookie) = headers.get("COOKIE") {
         if let Ok(cookie) = raw_cookie.to_str() {
-            ticket = tools::extract_auth_cookie(cookie, "PBSAuthCookie");
+            ticket = tools::extract_cookie(cookie, "PBSAuthCookie");
+            language = tools::extract_cookie(cookie, "PBSLangCookie");
         }
     }
 
@@ -455,7 +471,7 @@ fn extract_auth_data(headers: &http::HeaderMap) -> (Option<String>, Option<Strin
         _ => None,
     };
 
-    (ticket, token)
+    (ticket, token, language)
 }
 
 fn check_auth(
@@ -526,7 +542,7 @@ pub async fn handle_request(api: Arc<ApiConfig>, req: Request<Body>) -> Result<R
             ) {
                 // explicitly allow those calls without auth
             } else {
-                let (ticket, token) = extract_auth_data(&parts.headers);
+                let (ticket, token, _) = extract_auth_data(&parts.headers);
                 match check_auth(&method, &ticket, &token, &user_info) {
                     Ok(userid) => rpcenv.set_user(Some(userid.to_string())),
                     Err(err) => {
@@ -573,20 +589,20 @@ pub async fn handle_request(api: Arc<ApiConfig>, req: Request<Body>) -> Result<R
         }
 
         if comp_len == 0 {
-            let (ticket, token) = extract_auth_data(&parts.headers);
+            let (ticket, token, language) = extract_auth_data(&parts.headers);
             if ticket != None {
                 match check_auth(&method, &ticket, &token, &user_info) {
                     Ok(userid) => {
                         let new_token = assemble_csrf_prevention_token(csrf_secret(), &userid);
-                        return Ok(get_index(Some(userid), Some(new_token), &api, parts));
+                        return Ok(get_index(Some(userid), Some(new_token), language, &api, parts));
                     }
                     _ => {
                         tokio::time::delay_until(Instant::from_std(delay_unauth_time)).await;
-                        return Ok(get_index(None, None, &api, parts));
+                        return Ok(get_index(None, None, language, &api, parts));
                     }
                 }
             } else {
-                return Ok(get_index(None, None, &api, parts));
+                return Ok(get_index(None, None, language, &api, parts));
             }
         } else {
             let filename = api.find_alias(&components);

src/server/upid.rs

@@ -1,7 +1,6 @@
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use anyhow::{bail, Error};
-use chrono::Local;
 
 use proxmox::api::schema::{ApiStringFormat, Schema, StringSchema};
 use proxmox::const_regex;
@@ -89,7 +88,7 @@ impl UPID {
         Ok(UPID {
             pid,
             pstart: procfs::PidStat::read_from_pid(nix::unistd::Pid::from_raw(pid))?.starttime,
-            starttime: Local::now().timestamp(),
+            starttime: proxmox::tools::time::epoch_i64(),
             task_id,
             worker_type: worker_type.to_owned(),
             worker_id,

src/server/worker_task.rs

@@ -1,11 +1,11 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 use std::fs::File;
-use std::io::{BufRead, BufReader};
+use std::path::Path;
+use std::io::{Read, Write, BufRead, BufReader};
 use std::panic::UnwindSafe;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 
-use chrono::Local;
 use anyhow::{bail, format_err, Error};
 use futures::*;
 use lazy_static::lazy_static;
@@ -20,6 +20,7 @@ use proxmox::tools::fs::{create_path, open_file_locked, replace_file, CreateOpti
 
 use super::UPID;
 
+use crate::tools::logrotate::{LogRotate, LogRotateFiles};
 use crate::tools::FileLogger;
 use crate::api2::types::Userid;
 
@@ -32,6 +33,10 @@ pub const PROXMOX_BACKUP_LOG_DIR: &str = PROXMOX_BACKUP_LOG_DIR_M!();
 pub const PROXMOX_BACKUP_TASK_DIR: &str = PROXMOX_BACKUP_TASK_DIR_M!();
 pub const PROXMOX_BACKUP_TASK_LOCK_FN: &str = concat!(PROXMOX_BACKUP_TASK_DIR_M!(), "/.active.lock");
 pub const PROXMOX_BACKUP_ACTIVE_TASK_FN: &str = concat!(PROXMOX_BACKUP_TASK_DIR_M!(), "/active");
+pub const PROXMOX_BACKUP_INDEX_TASK_FN: &str = concat!(PROXMOX_BACKUP_TASK_DIR_M!(), "/index");
+pub const PROXMOX_BACKUP_ARCHIVE_TASK_FN: &str = concat!(PROXMOX_BACKUP_TASK_DIR_M!(), "/archive");
+
+const MAX_INDEX_TASKS: usize = 1000;
 
 lazy_static! {
     static ref WORKER_TASK_LIST: Mutex<HashMap<usize, Arc<WorkerTask>>> = Mutex::new(HashMap::new());
@@ -195,8 +200,8 @@ pub fn create_task_log_dirs() -> Result<(), Error> {
 /// If there is not a single line with at valid datetime, we assume the
 /// starttime to be the endtime
 pub fn upid_read_status(upid: &UPID) -> Result<TaskState, Error> {
-    let mut endtime = upid.starttime;
-    let mut status = TaskState::Unknown { endtime };
+
+    let mut status = TaskState::Unknown { endtime: upid.starttime };
 
     let path = upid.log_path();
 
@@ -207,22 +212,32 @@ pub fn upid_read_status(upid: &UPID) -> Result<TaskState, Error> {
     use std::io::SeekFrom;
     let _ = file.seek(SeekFrom::End(-8192)); // ignore errors
 
-    let reader = BufReader::new(file);
+    let mut data = Vec::with_capacity(8192);
+    file.read_to_end(&mut data)?;
 
-    for line in reader.lines() {
-        let line = line?;
+    // task logs should end with newline, we do not want it here
+    if data.len() > 0 && data[data.len()-1] == b'\n' {
+        data.pop();
+    }
 
-        let mut iter = line.splitn(2, ": ");
-        if let Some(time_str) = iter.next() {
-            endtime = chrono::DateTime::parse_from_rfc3339(time_str)
-                .map_err(|err| format_err!("cannot parse '{}': {}", time_str, err))?
-                .timestamp();
-        } else {
-            continue;
+    let last_line = {
+        let mut start = 0;
+        for pos in (0..data.len()).rev() {
+            if data[pos] == b'\n' {
+                start = data.len().min(pos + 1);
+                break;
+            }
         }
-        match iter.next().and_then(|rest| rest.strip_prefix("TASK ")) {
-            None => continue,
-            Some(rest) => {
+        &data[start..]
+    };
+
+    let last_line = std::str::from_utf8(last_line)
+        .map_err(|err| format_err!("upid_read_status: utf8 parse failed: {}", err))?;
+
+    let mut iter = last_line.splitn(2, ": ");
+    if let Some(time_str) = iter.next() {
+        if let Ok(endtime) = proxmox::tools::time::parse_rfc3339(time_str) {
+            if let Some(rest) = iter.next().and_then(|rest| rest.strip_prefix("TASK ")) {
                 if let Ok(state) = TaskState::from_endtime_and_message(endtime, rest) {
                     status = state;
                 }
@@ -316,85 +331,85 @@ pub struct TaskListInfo {
     pub state: Option<TaskState>, // endtime, status
 }
 
+fn lock_task_list_files(exclusive: bool) -> Result<std::fs::File, Error> {
+    let backup_user = crate::backup::backup_user()?;
+
+    let lock = open_file_locked(PROXMOX_BACKUP_TASK_LOCK_FN, std::time::Duration::new(10, 0), exclusive)?;
+    nix::unistd::chown(PROXMOX_BACKUP_TASK_LOCK_FN, Some(backup_user.uid), Some(backup_user.gid))?;
+
+    Ok(lock)
+}
+
+/// checks if the Task Archive is bigger that 'size_threshold' bytes, and
+/// rotates it if it is
+pub fn rotate_task_log_archive(size_threshold: u64, compress: bool, max_files: Option<usize>) -> Result<bool, Error> {
+    let _lock = lock_task_list_files(true)?;
+    let path = Path::new(PROXMOX_BACKUP_ARCHIVE_TASK_FN);
+    let metadata = path.metadata()?;
+    if metadata.len() > size_threshold {
+        let mut logrotate = LogRotate::new(PROXMOX_BACKUP_ARCHIVE_TASK_FN, compress).ok_or_else(|| format_err!("could not get archive file names"))?;
+        let backup_user = crate::backup::backup_user()?;
+        logrotate.rotate(
+            CreateOptions::new()
+                .owner(backup_user.uid)
+                .group(backup_user.gid),
+            max_files,
+        )?;
+        Ok(true)
+    } else {
+        Ok(false)
+    }
+}
+
 // atomically read/update the task list, update status of finished tasks
 // new_upid is added to the list when specified.
-// Returns a sorted list of known tasks,
-fn update_active_workers(new_upid: Option<&UPID>) -> Result<Vec<TaskListInfo>, Error> {
+fn update_active_workers(new_upid: Option<&UPID>) -> Result<(), Error> {
 
     let backup_user = crate::backup::backup_user()?;
 
-    let lock = open_file_locked(PROXMOX_BACKUP_TASK_LOCK_FN, std::time::Duration::new(10, 0))?;
-    nix::unistd::chown(PROXMOX_BACKUP_TASK_LOCK_FN, Some(backup_user.uid), Some(backup_user.gid))?;
+    let lock = lock_task_list_files(true)?;
 
-    let reader = match File::open(PROXMOX_BACKUP_ACTIVE_TASK_FN) {
-        Ok(f) => Some(BufReader::new(f)),
-        Err(err) => {
-            if err.kind() ==  std::io::ErrorKind::NotFound {
-                 None
-            } else {
-                bail!("unable to open active worker {:?} - {}", PROXMOX_BACKUP_ACTIVE_TASK_FN, err);
+    let mut finish_list: Vec<TaskListInfo> = read_task_file_from_path(PROXMOX_BACKUP_INDEX_TASK_FN)?;
+    let mut active_list: Vec<TaskListInfo> = read_task_file_from_path(PROXMOX_BACKUP_ACTIVE_TASK_FN)?
+        .into_iter()
+        .filter_map(|info| {
+            if info.state.is_some() {
+                // this can happen when the active file still includes finished tasks
+                finish_list.push(info);
+                return None;
             }
-        }
-    };
 
-    let mut active_list = vec![];
-    let mut finish_list = vec![];
-
-    if let Some(lines) = reader.map(|r| r.lines()) {
-
-        for line in lines {
-            let line = line?;
-            match parse_worker_status_line(&line) {
-                Err(err) => bail!("unable to parse active worker status '{}' - {}", line, err),
-                Ok((upid_str, upid, state)) => match state {
-                    None if worker_is_active_local(&upid) => {
-                        active_list.push(TaskListInfo { upid, upid_str, state: None });
-                    },
-                    None => {
-                        println!("Detected stopped UPID {}", upid_str);
-                        let status = upid_read_status(&upid)
-                            .unwrap_or_else(|_| TaskState::Unknown { endtime: Local::now().timestamp() });
-                        finish_list.push(TaskListInfo {
-                            upid, upid_str, state: Some(status)
-                        });
-                    },
-                    Some(status) => {
-                        finish_list.push(TaskListInfo {
-                            upid, upid_str, state: Some(status)
-                        })
-                    }
-                }
+            if !worker_is_active_local(&info.upid) {
+                println!("Detected stopped UPID {}", &info.upid_str);
+                let now = proxmox::tools::time::epoch_i64();
+                let status = upid_read_status(&info.upid)
+                    .unwrap_or_else(|_| TaskState::Unknown { endtime: now });
+                finish_list.push(TaskListInfo {
+                    upid: info.upid,
+                    upid_str: info.upid_str,
+                    state: Some(status)
+                });
+                return None;
             }
-        }
-    }
+
+            Some(info)
+        }).collect();
 
     if let Some(upid) = new_upid {
         active_list.push(TaskListInfo { upid: upid.clone(), upid_str: upid.to_string(), state: None });
     }
 
-    // assemble list without duplicates
-    // we include all active tasks,
-    // and fill up to 1000 entries with finished tasks
+    let active_raw = render_task_list(&active_list);
 
-    let max = 1000;
+    replace_file(
+        PROXMOX_BACKUP_ACTIVE_TASK_FN,
+        active_raw.as_bytes(),
+        CreateOptions::new()
+            .owner(backup_user.uid)
+            .group(backup_user.gid),
+    )?;
 
-    let mut task_hash = HashMap::new();
-
-    for info in active_list {
-        task_hash.insert(info.upid_str.clone(), info);
-    }
-
-    for info in finish_list {
-        if task_hash.len() > max { break; }
-        if !task_hash.contains_key(&info.upid_str) {
-            task_hash.insert(info.upid_str.clone(), info);
-        }
-    }
-
-    let mut task_list: Vec<TaskListInfo> = vec![];
-    for (_, info) in task_hash { task_list.push(info); }
-
-    task_list.sort_unstable_by(|b, a| { // lastest on top
+    finish_list.sort_unstable_by(|a, b| {
         match (&a.state, &b.state) {
             (Some(s1), Some(s2)) => s1.cmp(&s2),
             (Some(_), None) => std::cmp::Ordering::Less,
@@ -403,34 +418,198 @@ fn update_active_workers(new_upid: Option<&UPID>) -> Result<Vec<TaskListInfo>, E
         }
     });
 
-    let mut raw = String::new();
-    for info in &task_list {
-        if let Some(status) = &info.state {
-            raw.push_str(&format!("{} {:08X} {}\n", info.upid_str, status.endtime(), status));
-        } else {
-            raw.push_str(&info.upid_str);
-            raw.push('\n');
-        }
-    }
+
+    let start = if finish_list.len() > MAX_INDEX_TASKS {
+        finish_list.len() - MAX_INDEX_TASKS
+    } else {
+        0
+    };
+
+    let end = (start+MAX_INDEX_TASKS).min(finish_list.len());
+
+    let index_raw = if end > start {
+        render_task_list(&finish_list[start..end])
+    } else {
+        "".to_string()
+    };
 
     replace_file(
-        PROXMOX_BACKUP_ACTIVE_TASK_FN,
-        raw.as_bytes(),
+        PROXMOX_BACKUP_INDEX_TASK_FN,
+        index_raw.as_bytes(),
         CreateOptions::new()
             .owner(backup_user.uid)
             .group(backup_user.gid),
     )?;
 
+    if !finish_list.is_empty() && start > 0 {
+        match std::fs::OpenOptions::new().append(true).create(true).open(PROXMOX_BACKUP_ARCHIVE_TASK_FN) {
+            Ok(mut writer) => {
+                for info in &finish_list[0..start] {
+                    writer.write_all(render_task_line(&info).as_bytes())?;
+                }
+            },
+            Err(err) => bail!("could not write task archive - {}", err),
+        }
+
+        nix::unistd::chown(PROXMOX_BACKUP_ARCHIVE_TASK_FN, Some(backup_user.uid), Some(backup_user.gid))?;
+    }
+
     drop(lock);
 
-    Ok(task_list)
+    Ok(())
 }
 
-/// Returns a sorted list of known tasks
-///
-/// The list is sorted by `(starttime, endtime)` in ascending order
-pub fn read_task_list() -> Result<Vec<TaskListInfo>, Error> {
-    update_active_workers(None)
+fn render_task_line(info: &TaskListInfo) -> String {
+    let mut raw = String::new();
+    if let Some(status) = &info.state {
+        raw.push_str(&format!("{} {:08X} {}\n", info.upid_str, status.endtime(), status));
+    } else {
+        raw.push_str(&info.upid_str);
+        raw.push('\n');
+    }
+
+    raw
+}
+
+fn render_task_list(list: &[TaskListInfo]) -> String {
+    let mut raw = String::new();
+    for info in list {
+        raw.push_str(&render_task_line(&info));
+    }
+    raw
+}
+
+// note this is not locked, caller has to make sure it is
+// this will skip (and log) lines that are not valid status lines
+fn read_task_file<R: Read>(reader: R) -> Result<Vec<TaskListInfo>, Error>
+{
+    let reader = BufReader::new(reader);
+    let mut list = Vec::new();
+    for line in reader.lines() {
+        let line = line?;
+        match parse_worker_status_line(&line) {
+            Ok((upid_str, upid, state)) => list.push(TaskListInfo {
+                upid_str,
+                upid,
+                state
+            }),
+            Err(err) => {
+                eprintln!("unable to parse worker status '{}' - {}", line, err);
+                continue;
+            }
+        };
+    }
+
+    Ok(list)
+}
+
+// note this is not locked, caller has to make sure it is
+fn read_task_file_from_path<P>(path: P) -> Result<Vec<TaskListInfo>, Error>
+where
+    P: AsRef<std::path::Path> + std::fmt::Debug,
+{
+    let file = match File::open(&path) {
+        Ok(f) => f,
+        Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
+        Err(err) => bail!("unable to open task list {:?} - {}", path, err),
+    };
+
+    read_task_file(file)
+}
+
+enum TaskFile {
+    Active,
+    Index,
+    Archive,
+    End,
+}
+
+pub struct TaskListInfoIterator {
+    list: VecDeque<TaskListInfo>,
+    file: TaskFile,
+    archive: Option<LogRotateFiles>,
+    lock: Option<File>,
+}
+
+impl TaskListInfoIterator {
+    pub fn new(active_only: bool) -> Result<Self, Error> {
+        let (read_lock, active_list) = {
+            let lock = lock_task_list_files(false)?;
+            let active_list = read_task_file_from_path(PROXMOX_BACKUP_ACTIVE_TASK_FN)?;
+
+            let needs_update = active_list
+                .iter()
+                .any(|info| info.state.is_some() || !worker_is_active_local(&info.upid));
+
+            if needs_update {
+                drop(lock);
+                update_active_workers(None)?;
+                let lock = lock_task_list_files(false)?;
+                let active_list = read_task_file_from_path(PROXMOX_BACKUP_ACTIVE_TASK_FN)?;
+                (lock, active_list)
+            } else {
+                (lock, active_list)
+            }
+        };
+
+        let archive = if active_only {
+            None
+        } else {
+            let logrotate = LogRotate::new(PROXMOX_BACKUP_ARCHIVE_TASK_FN, true).ok_or_else(|| format_err!("could not get archive file names"))?;
+            Some(logrotate.files())
+        };
+
+        let file = if active_only { TaskFile::End } else { TaskFile::Active };
+        let lock = if active_only { None } else { Some(read_lock) };
+
+        Ok(Self {
+            list: active_list.into(),
+            file,
+            archive,
+            lock,
+        })
+    }
+}
+
+impl Iterator for TaskListInfoIterator {
+    type Item = Result<TaskListInfo, Error>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            if let Some(element) = self.list.pop_back() {
+                return Some(Ok(element));
+            } else {
+                match self.file {
+                    TaskFile::Active => {
+                        let index = match read_task_file_from_path(PROXMOX_BACKUP_INDEX_TASK_FN) {
+                            Ok(index) => index,
+                            Err(err) => return Some(Err(err)),
+                        };
+                        self.list.append(&mut index.into());
+                        self.file = TaskFile::Index;
+                    },
+                    TaskFile::Index | TaskFile::Archive => {
+                        if let Some(mut archive) = self.archive.take() {
+                            if let Some(file) = archive.next() {
+                                let list = match read_task_file(file) {
+                                    Ok(list) => list,
+                                    Err(err) => return Some(Err(err)),
+                                };
+                                self.list.append(&mut list.into());
+                                self.archive = Some(archive);
+                                self.file = TaskFile::Archive;
+                                continue;
+                            }
+                        }
+                        self.file = TaskFile::End;
+                        self.lock.take();
+                        return None;
+                    }
+                    TaskFile::End => return None,
+                }
+            }
+        }
+    }
 }
 
 /// Launch long running worker tasks.
@@ -577,7 +756,7 @@ impl WorkerTask {
     pub fn create_state(&self, result: &Result<(), Error>) -> TaskState {
         let warn_count = self.data.lock().unwrap().warn_count;
 
-        let endtime = Local::now().timestamp();
+        let endtime = proxmox::tools::time::epoch_i64();
 
         if let Err(err) = result {
             TaskState::Error { message: err.to_string(), endtime }

src/tools.rs

@@ -5,11 +5,9 @@ use std::any::Any;
 use std::collections::HashMap;
 use std::hash::BuildHasher;
 use std::fs::File;
-use std::io::{self, BufRead, ErrorKind, Read};
+use std::io::{self, BufRead, ErrorKind, Read, Seek, SeekFrom};
 use std::os::unix::io::RawFd;
 use std::path::Path;
-use std::time::Duration;
-use std::time::{SystemTime, SystemTimeError, UNIX_EPOCH};
 
 use anyhow::{bail, format_err, Error};
 use serde_json::Value;
@@ -34,6 +32,10 @@ pub mod ticket;
 pub mod statistics;
 pub mod systemd;
 pub mod nom;
+pub mod logrotate;
+
+mod parallel_handler;
+pub use parallel_handler::*;
 
 mod wrapped_reader_stream;
 pub use wrapped_reader_stream::*;
@@ -326,9 +328,9 @@ pub fn assert_if_modified(digest1: &str, digest2: &str) -> Result<(), Error> {
     Ok(())
 }
 
-/// Extract authentication cookie from cookie header.
+/// Extract a specific cookie from cookie header.
 /// We assume cookie_name is already url encoded.
-pub fn extract_auth_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
+pub fn extract_cookie(cookie: &str, cookie_name: &str) -> Option<String> {
     for pair in cookie.split(';') {
         let (name, value) = match pair.find('=') {
             Some(i) => (pair[..i].trim(), pair[(i + 1)..].trim()),
@@ -403,7 +405,7 @@ pub fn normalize_uri_path(path: &str) -> Result<(String, Vec<&str>), Error> {
 pub fn command_output(
     output: std::process::Output,
     exit_code_check: Option<fn(i32) -> bool>,
-) -> Result<String, Error> {
+) -> Result<Vec<u8>, Error> {
 
     if !output.status.success() {
         match output.status.code() {
@@ -424,8 +426,19 @@ pub fn command_output(
         }
     }
 
-    let output = String::from_utf8(output.stdout)?;
+    Ok(output.stdout)
+}
 
+/// Helper to check result from std::process::Command output, returns String.
+///
+/// The exit_code_check() function should return true if the exit code
+/// is considered successful.
+pub fn command_output_as_string(
+    output: std::process::Output,
+    exit_code_check: Option<fn(i32) -> bool>,
+) -> Result<String, Error> {
+    let output = command_output(output, exit_code_check)?;
+    let output = String::from_utf8(output)?;
     Ok(output)
 }
 
@@ -437,7 +450,7 @@ pub fn run_command(
    let output = command.output()
         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 
-    let output = crate::tools::command_output(output, exit_code_check)
+    let output = crate::tools::command_output_as_string(output, exit_code_check)
         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 
     Ok(output)
@@ -547,18 +560,6 @@ pub fn file_get_non_comment_lines<P: AsRef<Path>>(
     }))
 }
 
-pub fn epoch_now() -> Result<Duration, SystemTimeError> {
-    SystemTime::now().duration_since(UNIX_EPOCH)
-}
-
-pub fn epoch_now_f64() -> Result<f64, SystemTimeError> {
-    Ok(epoch_now()?.as_secs_f64())
-}
-
-pub fn epoch_now_u64() -> Result<u64, SystemTimeError> {
-    Ok(epoch_now()?.as_secs())
-}
-
 pub fn setup_safe_path_env() {
     std::env::set_var("PATH", "/sbin:/bin:/usr/sbin:/usr/bin");
     // Make %ENV safer - as suggested by https://perldoc.perl.org/perlsec.html
@@ -577,3 +578,32 @@ pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
         None => &[],
     }
 }
+
+/// Seeks to start of file and computes the SHA256 hash
+pub fn compute_file_csum(file: &mut File) -> Result<([u8; 32], u64), Error> {
+
+    file.seek(SeekFrom::Start(0))?;
+
+    let mut hasher = openssl::sha::Sha256::new();
+    let mut buffer = proxmox::tools::vec::undefined(256*1024);
+    let mut size: u64 = 0;
+
+    loop {
+        let count = match file.read(&mut buffer) {
+            Ok(count) => count,
+            Err(ref err) if err.kind() == std::io::ErrorKind::Interrupted => {
+                continue;
+            }
+            Err(err) => return Err(err.into()),
+        };
+        if count == 0 {
+            break;
+        }
+        size += count as u64;
+        hasher.update(&buffer[..count]);
+    }
+
+    let csum = hasher.finish();
+
+    Ok((csum, size))
+}

src/tools/file_logger.rs

@@ -1,5 +1,4 @@
 use anyhow::{Error};
-use chrono::{TimeZone, Local};
 use std::io::Write;
 
 /// Log messages with timestamps into files
@@ -56,7 +55,10 @@ impl FileLogger {
             stdout.write_all(b"\n").unwrap();
         }
 
-        let line = format!("{}: {}\n", Local.timestamp(Local::now().timestamp(), 0).to_rfc3339(), msg);
+        let now = proxmox::tools::time::epoch_i64();
+        let rfc3339 = proxmox::tools::time::epoch_to_rfc3339(now).unwrap();
+
+        let line = format!("{}: {}\n", rfc3339, msg);
         self.file.write_all(line.as_bytes()).unwrap();
     }
 }

src/tools/format.rs

@@ -1,6 +1,5 @@
 use anyhow::{Error};
 use serde_json::Value;
-use chrono::{Local, TimeZone};
 
 pub fn strip_server_file_expenstion(name: &str) -> String {
 
@@ -25,8 +24,12 @@ pub fn render_epoch(value: &Value, _record: &Value) -> Result<String, Error> {
     if value.is_null() { return Ok(String::new()); }
     let text = match value.as_i64() {
         Some(epoch) => {
-            Local.timestamp(epoch, 0).format("%c").to_string()
-        }
+            if let Ok(epoch_string) = proxmox::tools::time::strftime_local("%c", epoch as i64) {
+                epoch_string
+            } else {
+                epoch.to_string()
+            }
+        },
         None => {
             value.to_string()
         }

src/tools/logrotate.rs

@@ -0,0 +1,184 @@
+use std::path::{Path, PathBuf};
+use std::fs::{File, rename};
+use std::os::unix::io::FromRawFd;
+use std::io::Read;
+
+use anyhow::{bail, Error};
+use nix::unistd;
+
+use proxmox::tools::fs::{CreateOptions, make_tmp_file, replace_file};
+
+/// Used for rotating log files and iterating over them
+pub struct LogRotate {
+    base_path: PathBuf,
+    compress: bool,
+}
+
+impl LogRotate {
+    /// Creates a new instance if the path given is a valid file name
+    /// (iow. does not end with ..)
+    /// 'compress' decides if compresses files will be created on
+    /// rotation, and if it will search '.zst' files when iterating
+    pub fn new<P: AsRef<Path>>(path: P, compress: bool) -> Option<Self> {
+        if path.as_ref().file_name().is_some() {
+            Some(Self {
+                base_path: path.as_ref().to_path_buf(),
+                compress,
+            })
+        } else {
+            None
+        }
+    }
+
+    /// Returns an iterator over the logrotated file names that exist
+    pub fn file_names(&self) -> LogRotateFileNames {
+        LogRotateFileNames {
+            base_path: self.base_path.clone(),
+            count: 0,
+            compress: self.compress
+        }
+    }
+
+    /// Returns an iterator over the logrotated file handles
+    pub fn files(&self) -> LogRotateFiles {
+        LogRotateFiles {
+            file_names: self.file_names(),
+        }
+    }
+
+    /// Rotates the files up to 'max_files'
+    /// if the 'compress' option was given it will compress the newest file
+    ///
+    /// e.g. rotates
+    /// foo.2.zst => foo.3.zst
+    /// foo.1.zst => foo.2.zst
+    /// foo       => foo.1.zst
+    ///           => foo
+    pub fn rotate(&mut self, options: CreateOptions, max_files: Option<usize>) -> Result<(), Error> {
+        let mut filenames: Vec<PathBuf> = self.file_names().collect();
+        if filenames.is_empty() {
+            return Ok(()); // no file means nothing to rotate
+        }
+
+        let mut next_filename = self.base_path.clone().canonicalize()?.into_os_string();
+
+        if self.compress {
+            next_filename.push(format!(".{}.zst", filenames.len()));
+        } else {
+            next_filename.push(format!(".{}", filenames.len()));
+        }
+
+        filenames.push(PathBuf::from(next_filename));
+        let count = filenames.len();
+
+        // rotate all but the first, that we maybe have to compress
+        for i in (1..count-1).rev() {
+            rename(&filenames[i], &filenames[i+1])?;
+        }
+
+        if self.compress {
+            let mut source = File::open(&filenames[0])?;
+            let (fd, tmp_path) = make_tmp_file(&filenames[1], options.clone())?;
+            let target = unsafe { File::from_raw_fd(fd) };
+            let mut encoder = match zstd::stream::write::Encoder::new(target, 0) {
+                Ok(encoder) => encoder,
+                Err(err) => {
+                    let _ = unistd::unlink(&tmp_path);
+                    bail!("creating zstd encoder failed - {}", err);
+                }
+            };
+
+            if let Err(err) = std::io::copy(&mut source, &mut encoder) {
+                let _ = unistd::unlink(&tmp_path);
+                bail!("zstd encoding failed for file {:?} - {}", &filenames[1], err);
+            }
+
+            if let Err(err) = encoder.finish() {
+                let _ = unistd::unlink(&tmp_path);
+                bail!("zstd finish failed for file {:?} - {}", &filenames[1], err);
+            }
+
+            if let Err(err) = rename(&tmp_path, &filenames[1]) {
+                let _ = unistd::unlink(&tmp_path);
+                bail!("rename failed for file {:?} - {}", &filenames[1], err);
+            }
+
+            unistd::unlink(&filenames[0])?;
+        } else {
+            rename(&filenames[0], &filenames[1])?;
+        }
+
+        // create empty original file
+        replace_file(&filenames[0], b"", options)?;
+
+        if let Some(max_files) = max_files {
+            // delete all files > max_files
+            for file in filenames.iter().skip(max_files) {
+                if let Err(err) = unistd::unlink(file) {
+                    eprintln!("could not remove {:?}: {}", &file, err);
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+/// Iterator over logrotated file names
+pub struct LogRotateFileNames {
+    base_path: PathBuf,
+    count: usize,
+    compress: bool,
+}
+
+impl Iterator for LogRotateFileNames {
+    type Item = PathBuf;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.count > 0 {
+            let mut path: std::ffi::OsString = self.base_path.clone().into();
+
+            path.push(format!(".{}", self.count));
+            self.count += 1;
+
+            if Path::new(&path).is_file() {
+                Some(path.into())
+            } else if self.compress {
+                path.push(".zst");
+                if Path::new(&path).is_file() {
+                    Some(path.into())
+                } else {
+                    None
+                }
+            } else {
+                None
+            }
+        } else if self.base_path.is_file() {
+            self.count += 1;
+            Some(self.base_path.to_path_buf())
+        } else {
+            None
+        }
+    }
+}
+
+/// Iterator over logrotated files by returning a boxed reader
+pub struct LogRotateFiles {
+    file_names: LogRotateFileNames,
+}
+
+impl Iterator for LogRotateFiles {
+    type Item = Box<dyn Read + Send>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let filename = self.file_names.next()?;
+        let file = File::open(&filename).ok()?;
+
+        if filename.extension().unwrap_or(std::ffi::OsStr::new("")) == "zst" {
+            let encoder = zstd::stream::read::Decoder::new(file).ok()?;
+            return Some(Box::new(encoder));
+        }
+
+        Some(Box::new(file))
+    }
+}