bump version to 1.0.1-1

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

Cargo.toml

@@ -1,7 +1,16 @@
 [package]
 name = "proxmox-backup"
-version = "0.9.5"
+version = "1.0.1"
-authors = ["Dietmar Maurer <dietmar@proxmox.com>"]
+authors = [
+    "Dietmar Maurer <dietmar@proxmox.com>",
+    "Dominik Csapak <d.csapak@proxmox.com>",
+    "Christian Ebner <c.ebner@proxmox.com>",
+    "Fabian Grünbichler <f.gruenbichler@proxmox.com>",
+    "Stefan Reiter <s.reiter@proxmox.com>",
+    "Thomas Lamprecht <t.lamprecht@proxmox.com>",
+    "Wolfgang Bumiller <w.bumiller@proxmox.com>",
+    "Proxmox Support Team <support@proxmox.com>",
+]
 edition = "2018"
 license = "AGPL-3"
 description = "Proxmox Backup"

debian/changelog

@@ -1,3 +1,102 @@
+rust-proxmox-backup (1.0.1-1) unstable; urgency=medium
+
+  * ui: datastore summary: drop 'removed bytes' display
+
+  * ui: datastore add: set default schedule
+
+  * prune sim: make backup schedule a form, bind update button to its validity
+
+  * daemon: add workaround for race in reloading and systemd 'ready' notification
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 11 Nov 2020 10:18:12 +0100
+
+rust-proxmox-backup (1.0.0-1) unstable; urgency=medium
+
+  * fix #3121: forbid removing used remotes
+
+  * docs: backup-client: encryption: discuss paperkey command
+
+  * pxar: log when skipping mount points
+
+  * ui: show also parent ACLs which affect a datastore in its panel
+
+  * api: node/apt: add versions call
+
+  * ui: make Datastore a selectable panel again. Show a datastore summary
+    list, and provide unfiltered access to all sync and verify jobs.
+
+  * ui: add help tool-button to various paneös
+
+  * ui: set various onlineHelp buttons
+
+  * zfs: mount new zpools created via API under /mnt/datastore/<id>
+
+  * ui: move disks/directory views to its own tab panel
+
+  * fix #3060: continue sync if we cannot aquire the group lock
+
+  * HttpsConnector: include destination on connect errors
+
+  * fix #3060:: improve get_owner error handling
+
+  * remote.cfg: rename userid to 'auth-id'
+
+  * verify: log/warn on invalid owner
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 10 Nov 2020 14:36:13 +0100
+
+rust-proxmox-backup (0.9.7-1) unstable; urgency=medium
+
+  * ui: add remote store selector
+
+  * tools/daemon: fix reload with open connections
+
+  * pxar/create: fix endless loop for shrinking files
+
+  * pxar/create: handle ErrorKind::Interrupted for file reads
+
+  * ui: add action-button for changing backup group owner
+
+  * docs: add interactive prune simulator
+
+  * verify: fix unprivileged verification jobs
+
+  * tasks: allow access to job tasks
+
+  * drop internal 'backup@pam' owner, sync jobs need to set a explicit owner
+
+  * api: datastore: allow to set "verify-new" option over API
+
+  * ui: datastore: add Options tab, allowing one to change per-datastore
+    notification and verify-new options
+
+  * docs: scroll navigation bar to current active section
+
+ -- Proxmox Support Team <support@proxmox.com>  Mon, 09 Nov 2020 07:36:58 +0100
+
+rust-proxmox-backup (0.9.6-1) unstable; urgency=medium
+
+  * fix #3106: improve queueing new incoming connections
+
+  * fix #2870: sync: ensure a updated ticket is used, if available
+
+  * proxy: log if there are too many open connections
+
+  * ui: SyncJobEdit: fix sending 'delete' values on SyncJob creation
+
+  * datastore config: allow to configure who receives job notify emails
+
+  * ui: fix task description for log rotate
+
+  * proxy: also rotate auth.log file
+
+  * ui: add shell panel under administration
+
+  * ui: sync jobs: only set default schedule when creating new jobs and some
+    other small fixes
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 04 Nov 2020 19:12:57 +0100
+
 rust-proxmox-backup (0.9.5-1) unstable; urgency=medium
 
   * ui: user menu: allow one to change the language while staying logged in

debian/control

@@ -104,7 +104,9 @@ Depends: fonts-font-awesome,
          libjs-extjs (>= 6.0.1),
          libzstd1 (>= 1.3.8),
          lvm2,
+         openssh-server,
          pbs-i18n,
+         postfix | mail-transport-agent,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
          proxmox-widget-toolkit (>= 2.3-6),
@@ -113,6 +115,7 @@ Depends: fonts-font-awesome,
          ${misc:Depends},
          ${shlibs:Depends},
 Recommends: zfsutils-linux,
+            ifupdown2,
 Description: Proxmox Backup Server daemon with tools and GUI
  This package contains the Proxmox Backup Server daemons and related
  tools. This includes a web-based graphical user interface.

debian/control.in

@@ -4,7 +4,9 @@ Depends: fonts-font-awesome,
          libjs-extjs (>= 6.0.1),
          libzstd1 (>= 1.3.8),
          lvm2,
+         openssh-server,
          pbs-i18n,
+         postfix | mail-transport-agent,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
          proxmox-widget-toolkit (>= 2.3-6),
@@ -13,6 +15,7 @@ Depends: fonts-font-awesome,
          ${misc:Depends},
          ${shlibs:Depends},
 Recommends: zfsutils-linux,
+            ifupdown2,
 Description: Proxmox Backup Server daemon with tools and GUI
  This package contains the Proxmox Backup Server daemons and related
  tools. This includes a web-based graphical user interface.

debian/lintian-overrides

@@ -1,2 +1,2 @@
-proxmox-backup-server: package-installs-apt-sources etc/apt/sources.list.d/pbstest-beta.list
+proxmox-backup-server: package-installs-apt-sources etc/apt/sources.list.d/pbs-enterprise.list
 proxmox-backup-server: systemd-service-file-refers-to-unusual-wantedby-target lib/systemd/system/proxmox-backup-banner.service getty.target

debian/postinst

@@ -25,6 +25,18 @@ case "$1" in
 				    sed -i '/^\s\+verify-schedule /d' /etc/proxmox-backup/datastore.cfg || true
 			fi
 		fi
+		if dpkg --compare-versions "$2" 'le' '0.9.5-1'; then
+			chown --quiet backup:backup /var/log/proxmox-backup/api/auth.log || true
+		fi
+		if dpkg --compare-versions "$2" 'le' '0.9.7-1'; then
+			if [ -e /etc/proxmox-backup/remote.cfg ]; then
+				echo "NOTE: Switching over remote.cfg to new field names.."
+				flock -w 30 /etc/proxmox-backup/.remote.lck \
+				    sed -i \
+				        -e 's/^\s\+userid /\tauth-id /g' \
+				        /etc/proxmox-backup/remote.cfg || true
+			fi
+		fi
 	fi
 	# FIXME: Remove in future version once we're sure no broken entries remain in anyone's files
 	if grep -q -e ':termproxy::[^@]\+: ' /var/log/proxmox-backup/tasks/active; then

debian/proxmox-backup-docs.links

@@ -1 +1,2 @@
 /usr/share/doc/proxmox-backup/proxmox-backup.pdf /usr/share/doc/proxmox-backup/html/proxmox-backup.pdf
+/usr/share/javascript/extjs /usr/share/doc/proxmox-backup/html/prune-simulator/extjs

debian/proxmox-backup-server.install

@@ -3,7 +3,7 @@ etc/proxmox-backup.service /lib/systemd/system/
 etc/proxmox-backup-banner.service /lib/systemd/system/
 etc/proxmox-backup-daily-update.service /lib/systemd/system/
 etc/proxmox-backup-daily-update.timer /lib/systemd/system/
-etc/pbstest-beta.list /etc/apt/sources.list.d/
+etc/pbs-enterprise.list /etc/apt/sources.list.d/
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-api
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-proxy
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-banner

debian/proxmox-backup-server.maintscript

@@ -0,0 +1 @@
+rm_conffile /etc/apt/sources.list.d/pbstest-beta.list 1.0.0~ proxmox-backup-server

docs/Makefile

@@ -14,6 +14,10 @@ MANUAL_PAGES := 			\
 	proxmox-backup-client.1		\
 	proxmox-backup-manager.1
 
+PRUNE_SIMULATOR_FILES := 					\
+	prune-simulator/index.html				\
+	prune-simulator/documentation.html			\
+	prune-simulator/prune-simulator.js
 
 # Sphinx documentation setup
 SPHINXOPTS    =
@@ -74,10 +78,11 @@ onlinehelpinfo:
 	@echo "Build finished. OnlineHelpInfo.js is in $(BUILDDIR)/scanrefs."
 
 .PHONY: html
-html: ${GENERATED_SYNOPSIS} images/proxmox-logo.svg custom.css conf.py
+html: ${GENERATED_SYNOPSIS} images/proxmox-logo.svg custom.css conf.py ${PRUNE_SIMULATOR_FILES}
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
-	cp images/proxmox-logo.svg $(BUILDDIR)/html/_static/
+	install -m 0644 custom.js custom.css images/proxmox-logo.svg $(BUILDDIR)/html/_static/
-	cp custom.css $(BUILDDIR)/html/_static/
+	install -dm 0755 $(BUILDDIR)/html/prune-simulator
+	install -m 0644 ${PRUNE_SIMULATOR_FILES} $(BUILDDIR)/html/prune-simulator
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 

docs/_ext/proxmox-scanrefs.py

@@ -44,7 +44,7 @@ def scan_extjs_files(wwwdir="../www"): # a bit rough i know, but we can optimize
                 js_files.append(os.path.join(root, filename))
     for js_file in js_files:
         fd = open(js_file).read()
-        allmatch = re.findall("onlineHelp:\s*[\'\"](.*?)[\'\"]", fd, re.M)
+        allmatch = re.findall("(?:onlineHelp:|get_help_tool\s*\()\s*[\'\"](.*?)[\'\"]", fd, re.M)
         for match in allmatch:
             anchor = match
             anchor = re.sub('_', '-', anchor) # normalize labels
@@ -73,7 +73,9 @@ class ReflabelMapper(Builder):
             'link': '/docs/index.html',
             'title': 'Proxmox Backup Server Documentation Index',
         }
-        self.env.used_anchors = scan_extjs_files()
+        # Disabled until we find a sensible way to scan proxmox-widget-toolkit
+        # as well
+        #self.env.used_anchors = scan_extjs_files()
 
         if not os.path.isdir(self.outdir):
             os.mkdir(self.outdir)
@@ -93,6 +95,9 @@ class ReflabelMapper(Builder):
                     logger.info('traversing section {}'.format(title.astext()))
                     ref_name = getattr(title, 'rawsource', title.astext())
 
+                    if (ref_name[:7] == ':term:`'):
+                        ref_name = ref_name[7:-1]
+
                     self.env.online_help[labelid] = {'link': '', 'title': ''}
                     self.env.online_help[labelid]['link'] = "/docs/" + os.path.basename(filename_html) + "#{}".format(labelid)
                     self.env.online_help[labelid]['title'] = ref_name
@@ -112,15 +117,18 @@ class ReflabelMapper(Builder):
     def validate_anchors(self):
         #pprint(self.env.online_help)
         to_remove = []
-        for anchor in self.env.used_anchors:
+
-            if anchor not in self.env.online_help:
+        # Disabled until we find a sensible way to scan proxmox-widget-toolkit
-                logger.info("[-] anchor {} is missing from onlinehelp!".format(anchor))
+        # as well
-        for anchor in self.env.online_help:
+        #for anchor in self.env.used_anchors:
-            if anchor not in self.env.used_anchors and anchor != 'pbs_documentation_index':
+        #    if anchor not in self.env.online_help:
-                logger.info("[*] anchor {} not used! deleting...".format(anchor))
+        #        logger.info("[-] anchor {} is missing from onlinehelp!".format(anchor))
-                to_remove.append(anchor)
+        #for anchor in self.env.online_help:
-        for anchor in to_remove:
+        #    if anchor not in self.env.used_anchors and anchor != 'pbs_documentation_index':
-            self.env.online_help.pop(anchor, None)
+        #        logger.info("[*] anchor {} not used! deleting...".format(anchor))
+        #        to_remove.append(anchor)
+        #for anchor in to_remove:
+        #   self.env.online_help.pop(anchor, None)
         return
 
     def finish(self):

docs/backup-client.rst

@@ -365,9 +365,22 @@ To set up a master key:
   backed up. It can happen, for example, that you back up an entire system, using
   a key on that system. If the system then becomes inaccessible for any reason
   and needs to be restored, this will not be possible as the encryption key will be
-  lost along with the broken system. In preparation for the worst case scenario,
+  lost along with the broken system.
-  you should consider keeping a paper copy of this key locked away in
+
-  a safe place.
+It is recommended that you keep your master key safe, but easily accessible, in
+order for quick disaster recovery. For this reason, the best place to store it
+is in your password manager, where it is immediately recoverable. As a backup to
+this, you should also save the key to a USB drive and store that in a secure
+place. This way, it is detached from any system, but is still easy to recover
+from, in case of emergency. Finally, in preparation for the worst case scenario,
+you should also consider keeping a paper copy of your master key locked away in
+a safe place. The ``paperkey`` subcommand can be used to create a QR encoded
+version of your master key. The following command sends the output of the
+``paperkey`` command to a text file, for easy printing.
+
+.. code-block:: console
+
+  proxmox-backup-client key paperkey --output-format text > qrkey.txt
 
 
 Restoring Data
@@ -535,6 +548,29 @@ To remove the ticket, issue a logout:
   # proxmox-backup-client logout
 
 
+.. _changing-backup-owner:
+
+Changing the Owner of a Backup Group
+------------------------------------
+
+By default, the owner of a backup group is the user which was used to originally
+create that backup group (or in the case of sync jobs, ``root@pam``). This
+means that if a user ``mike@pbs`` created a backup, another user ``john@pbs``
+can not be used to create backups in that same backup group.  In case you want
+to change the owner of a backup, you can do so with the below command, using a
+user that has ``Datastore.Modify`` privileges on the datastore.
+
+.. code-block:: console
+
+  # proxmox-backup-client change-owner vm/103 john@pbs
+
+This can also be done from within the web interface, by navigating to the
+`Content` section of the datastore that contains the backup group and
+selecting the user icon under the `Actions` column. Common cases for this could
+be to change the owner of a sync job from ``root@pam``, or to repurpose a
+backup group.
+
+
 .. _backup-pruning:
 
 Pruning and Removing Backups

docs/conf.py

@@ -171,6 +171,7 @@ html_theme_options = {
     'extra_nav_links': {
         'Proxmox Homepage': 'https://proxmox.com',
         'PDF': 'proxmox-backup.pdf',
+        'Prune Simulator' : 'prune-simulator/index.html',
     },
 
     'sidebar_width': '320px',
@@ -228,6 +229,10 @@ html_favicon = 'images/favicon.ico'
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+html_js_files = [
+    'custom.js',
+]
+
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.

docs/custom.js

@@ -0,0 +1,7 @@
+window.addEventListener('DOMContentLoaded', (event) => {
+    let activeSection = document.querySelector("a.current");
+    if (activeSection) {
+        // https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollIntoView
+        activeSection.scrollIntoView({ block: 'center' });
+    }
+});

docs/faq.rst

@@ -27,7 +27,7 @@ How long will my Proxmox Backup Server version be supported?
 +-----------------------+--------------------+---------------+------------+--------------------+
 |Proxmox Backup Version | Debian Version     | First Release | Debian EOL | Proxmox Backup EOL |
 +=======================+====================+===============+============+====================+
-|Proxmox Backup 1.x     | Debian 10 (Buster) | tba           | tba        | tba                |
+|Proxmox Backup 1.x     | Debian 10 (Buster) | 2020-11       | tba        | tba                |
 +-----------------------+--------------------+---------------+------------+--------------------+
 
 

docs/gui.rst

@@ -4,7 +4,7 @@ Graphical User Interface
 Proxmox Backup Server offers an integrated, web-based interface to manage the
 server. This means that you can carry out all administration tasks through your
 web browser, and that you don't have to worry about installing extra management
-tools. The web interface also provides a built in console, so if you prefer the
+tools. The web interface also provides a built-in console, so if you prefer the
 command line or need some extra control, you have this option.
 
 The web interface can be accessed via https://youripaddress:8007. The default
@@ -28,7 +28,6 @@ Login
 -----
 
 .. image:: images/screenshots/pbs-gui-login-window.png
-  :width: 250
   :align: right
   :alt: PBS login window
 
@@ -44,14 +43,13 @@ GUI Overview
 ------------
 
 .. image:: images/screenshots/pbs-gui-dashboard.png
-  :width: 250
   :align: right
   :alt: PBS GUI Dashboard
 
 The Proxmox Backup Server web interface consists of 3 main sections:
 
 * **Header**: At the top. This shows version information, and contains buttons to view
-  documentation, monitor running tasks, and logout.
+  documentation, monitor running tasks, set the language and logout.
 * **Sidebar**: On the left. This contains the configuration options for
   the server.
 * **Configuration Panel**: In the center. This contains the control interface for the
@@ -79,18 +77,17 @@ Configuration
 The Configuration section contains some system configuration options, such as
 time and network configuration. It also contains the following subsections:
 
-* **User Management**: Add users and manage accounts
+* **Access Control**: Add and manage users, API tokens, and the permissions
-* **Permissions**: Manage permissions for various users
+  associated with these items
 * **Remotes**: Add, edit and remove remotes (see :term:`Remote`)
-* **Sync Jobs**: Manage and run sync jobs to remotes
+* **Subscription**: Upload a subscription key, view subscription status and
-* **Subscription**: Upload a subscription key and view subscription status
+  access a text-based system report.
 
 
 Administration
 ^^^^^^^^^^^^^^
 
 .. image:: images/screenshots/pbs-gui-administration-serverstatus.png
-  :width: 250
   :align: right
   :alt: Administration: Server Status overview
 
@@ -105,7 +102,6 @@ tasks and information. These are:
 * **Tasks**: Task history with multiple filter options
 
 .. image:: images/screenshots/pbs-gui-disks.png
-  :width: 250
   :align: right
   :alt: Administration: Disks
 
@@ -120,16 +116,21 @@ The administration menu item also contains a disk management subsection:
 Datastore
 ^^^^^^^^^
 
-.. image:: images/screenshots/pbs-gui-datastore.png
+.. image:: images/screenshots/pbs-gui-datastore-summary.png
-  :width: 250
   :align: right
   :alt: Datastore Configuration
 
-The Datastore section provides an interface for creating and managing
+The Datastore section contains interfaces for creating and managing
-datastores. It contains a subsection for each datastore on the system, in
+datastores. It contains a button to create a new datastore on the server, as
-which you can use the top panel to view:
+well as a subsection for each datastore on the system, in which you can use the
+top panel to view:
 
+* **Summary**: Access a range of datastore usage statistics
 * **Content**: Information on the datastore's backup groups and their respective
   contents
-* **Statistics**: Usage statistics for the datastore
+* **Prune & GC**: Schedule :ref:`pruning <backup-pruning>` and :ref:`garbage
-* **Permissions**: View and manage permissions for the datastore
+  collection <garbage-collection>` operations, and run garbage collection
+  manually
+* **Sync Jobs**: Create, manage and run :ref:`syncjobs` from remote servers
+* **Verify Jobs**: Create, manage and run :ref:`maintenance_verification` jobs on the
+  datastore

docs/installation.rst

@@ -9,7 +9,7 @@ Debian_ from the provided package repository.
 
 .. include:: package-repositories.rst
 
-Server installation
+Server Installation
 -------------------
 
 The backup server stores the actual backed up data and provides a web based GUI
@@ -37,22 +37,21 @@ Download the ISO from |DOWNLOADS|.
 It includes the following:
 
 * The `Proxmox Backup`_ server installer, which partitions the local
-  disk(s) with ext4, ext3, xfs or ZFS, and installs the operating
+  disk(s) with ext4, xfs or ZFS, and installs the operating system
-  system
 
 * Complete operating system (Debian Linux, 64-bit)
 
-* Our Linux kernel with ZFS support
+* Proxmox Linux kernel with ZFS support
 
 * Complete tool-set to administer backups and all necessary resources
 
-* Web based GUI management interface
+* Web based management interface
 
 .. note:: During the installation process, the complete server
    is used by default and all existing data is removed.
 
 
-Install `Proxmox Backup`_ server on Debian
+Install `Proxmox Backup`_ Server on Debian
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Proxmox ships as a set of Debian packages which can be installed on top of a
@@ -84,11 +83,11 @@ support, and a set of common and useful packages.
   when LVM_ or ZFS_ is used. The network configuration is completely up to you
   as well.
 
-.. note:: You can access the web interface of the Proxmox Backup Server with
+.. Note:: You can access the web interface of the Proxmox Backup Server with
-  your web browser, using HTTPS on port 8007. For example at
+   your web browser, using HTTPS on port 8007. For example at
- ``https://<ip-or-dns-name>:8007``
+   ``https://<ip-or-dns-name>:8007``
 
-Install Proxmox Backup server on `Proxmox VE`_
+Install Proxmox Backup Server on `Proxmox VE`_
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 After configuring the
@@ -104,14 +103,14 @@ After configuring the
    server to store backups. Should the hypervisor server fail, you can
    still access the backups.
 
-.. note::
+.. Note:: You can access the web interface of the Proxmox Backup Server with
-  You can access the web interface of the Proxmox Backup Server with your web
+   your web browser, using HTTPS on port 8007. For example at
-  browser, using HTTPS on port 8007. For example at ``https://<ip-or-dns-name>:8007``
+   ``https://<ip-or-dns-name>:8007``
 
-Client installation
+Client Installation
 -------------------
 
-Install `Proxmox Backup`_ client on Debian
+Install `Proxmox Backup`_ Client on Debian
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Proxmox ships as a set of Debian packages to be installed on

docs/introduction.rst

@@ -127,8 +127,7 @@ language.
 
  -- `The Rust Programming Language <https://doc.rust-lang.org/book/ch00-00-introduction.html>`_
 
-.. todo:: further explain the software stack
+.. _get_help:
-
 
 Getting Help
 ------------

docs/maintenance.rst

@@ -1,13 +1,184 @@
 Maintenance Tasks
 =================
 
+.. _maintenance_pruning:
+
+Pruning
+-------
+
+Prune lets you specify which backup snapshots you want to keep. The
+following retention options are available:
+
+``keep-last <N>``
+  Keep the last ``<N>`` backup snapshots.
+
+``keep-hourly <N>``
+  Keep backups for the last ``<N>`` hours. If there is more than one
+  backup for a single hour, only the latest is kept.
+
+``keep-daily <N>``
+  Keep backups for the last ``<N>`` days. If there is more than one
+  backup for a single day, only the latest is kept.
+
+``keep-weekly <N>``
+  Keep backups for the last ``<N>`` weeks. If there is more than one
+  backup for a single week, only the latest is kept.
+
+  .. note:: Weeks start on Monday and end on Sunday. The software
+     uses the `ISO week date`_ system and handles weeks at
+     the end of the year correctly.
+
+``keep-monthly <N>``
+  Keep backups for the last ``<N>`` months. If there is more than one
+  backup for a single month, only the latest is kept.
+
+``keep-yearly <N>``
+  Keep backups for the last ``<N>`` years. If there is more than one
+  backup for a single year, only the latest is kept.
+
+The retention options are processed in the order given above. Each option
+only covers backups within its time period. The next option does not take care
+of already covered backups. It will only consider older backups.
+
+Unfinished and incomplete backups will be removed by the prune command unless
+they are newer than the last successful backup. In this case, the last failed
+backup is retained.
+
+Prune Simulator
+^^^^^^^^^^^^^^^
+
+You can use the built-in `prune simulator <prune-simulator/index.html>`_
+to explore the effect of different retetion options with various backup
+schedules.
+
+Manual Pruning
+^^^^^^^^^^^^^^
+
+.. image:: images/screenshots/pbs-gui-datastore-content-prune-group.png
+  :target: _images/pbs-gui-datastore-content-prune-group.png
+  :align: right
+  :alt: Prune and garbage collection options
+
+To access pruning functionality for a specific backup group, you can use the
+prune command line option discussed in :ref:`backup-pruning`, or navigate to
+the **Content** tab of the datastore and click the scissors icon in the
+**Actions** column of the relevant backup group.
+
+Prune Schedules
+^^^^^^^^^^^^^^^
+
+To prune on a datastore level, scheduling options can be found under the
+**Prune & GC** tab of the datastore. Here you can set retention settings and
+edit the interval at which pruning takes place.
+
+.. image:: images/screenshots/pbs-gui-datastore-prunegc.png
+  :target: _images/pbs-gui-datastore-prunegc.png
+  :align: right
+  :alt: Prune and garbage collection options
+
+
+Retention Settings Example
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The backup frequency and retention of old backups may depend on how often data
+changes, and how important an older state may be, in a specific work load.
+When backups act as a company's document archive, there may also be legal
+requirements for how long backup snapshots must be kept.
+
+For this example, we assume that you are doing daily backups, have a retention
+period of 10 years, and the period between backups stored gradually grows.
+
+- **keep-last:** ``3`` - even if only daily backups, an admin may want to create
+  an extra one just before or after a big upgrade. Setting keep-last ensures
+  this.
+
+- **keep-hourly:** not set - for daily backups this is not relevant. You cover
+  extra manual backups already, with keep-last.
+
+- **keep-daily:** ``13`` - together with keep-last, which covers at least one
+  day, this ensures that you have at least two weeks of backups.
+
+- **keep-weekly:** ``8`` - ensures that you have at least two full months of
+  weekly backups.
+
+- **keep-monthly:** ``11`` - together with the previous keep settings, this
+  ensures that you have at least a year of monthly backups.
+
+- **keep-yearly:** ``9`` - this is for the long term archive. As you covered the
+  current year with the previous options, you would set this to nine for the
+  remaining ones, giving you a total of at least 10 years of coverage.
+
+We recommend that you use a higher retention period than is minimally required
+by your environment; you can always reduce it if you find it is unnecessarily
+high, but you cannot recreate backup snapshots from the past.
+
+
+.. _maintenance_gc:
+
 Garbage Collection
 ------------------
 
 You can monitor and run :ref:`garbage collection <garbage-collection>` on the
 Proxmox Backup Server using the ``garbage-collection`` subcommand of
-``proxmox-backup-manager``. You can use the ``start`` subcommand to manually start garbage
+``proxmox-backup-manager``. You can use the ``start`` subcommand to manually
-collection on an entire datastore and the ``status`` subcommand to see
+start garbage collection on an entire datastore and the ``status`` subcommand to
-attributes relating to the :ref:`garbage collection <garbage-collection>`.
+see attributes relating to the :ref:`garbage collection <garbage-collection>`.
 
-.. todo:: Add section on verification
+This functionality can also be accessed in the GUI, by navigating to **Prune &
+GC** from the top panel. From here, you can edit the schedule at which garbage
+collection runs and manually start the operation.
+
+
+.. _maintenance_verification:
+
+Verification
+------------
+
+.. image:: images/screenshots/pbs-gui-datastore-verifyjob-add.png
+  :target: _images/pbs-gui-datastore-verifyjob-add.png
+  :align: right
+  :alt: Adding a verify job
+
+Proxmox Backup offers various verification options to ensure that backup data is
+intact.  Verification is generally carried out through the creation of verify
+jobs. These are scheduled tasks that run verification at a given interval (see
+:ref:`calendar-events`). With these, you can set whether already verified
+snapshots are ignored, as well as set a time period, after which verified jobs
+are checked again. The interface for creating verify jobs can be found under the
+**Verify Jobs** tab of the datastore.
+
+.. Note:: It is recommended that you reverify all backups at least monthly, even
+  if a previous verification was successful. This is becuase physical drives
+  are susceptible to damage over time, which can cause an old, working backup
+  to become corrupted in a process known as `bit rot/data degradation
+  <https://en.wikipedia.org/wiki/Data_degradation>`_. It is good practice to
+  have a regularly recurring (hourly/daily) verification job, which checks new
+  and expired backups, then another weekly/monthly job that will reverify
+  everything. This way, there will be no surprises when it comes to restoring
+  data.
+
+Aside from using verify jobs, you can also run verification manually on entire
+datastores, backup groups, or snapshots. To do this, navigate to the **Content**
+tab of the datastore and either click *Verify All*, or select the *V.* icon from
+the *Actions* column in the table.
+
+.. _maintenance_notification:
+
+Notifications
+-------------
+
+Proxmox Backup Server can send you notification emails about automatically
+scheduled verification, garbage-collection and synchronization tasks results.
+
+By default, notifications are send to the email address configured for the
+`root@pam` user. You can set that user for each datastore.
+
+You can also change the level of notification received per task type, the
+following options are available:
+
+* Always: send a notification for any scheduled task, independent of the
+  outcome
+
+* Errors: send a notification for any scheduled task resulting in an error
+
+* Never: do not send any notification at all

docs/managing-remotes.rst

@@ -59,13 +59,13 @@ Sync Jobs
   :alt: Add a Sync Job
 
 Sync jobs are configured to pull the contents of a datastore on a **Remote** to
-a local datastore. You can manage sync jobs under **Configuration -> Sync Jobs**
+a local datastore. You can manage sync jobs in the web interface, from the
-in the web interface, or using the ``proxmox-backup-manager sync-job`` command.
+**Sync Jobs** tab of the datastore which you'd like to set one up for, or using
-The configuration information for sync jobs is stored at
+the ``proxmox-backup-manager sync-job`` command.  The configuration information
-``/etc/proxmox-backup/sync.cfg``. To create a new sync job, click the add button
+for sync jobs is stored at ``/etc/proxmox-backup/sync.cfg``. To create a new
-in the GUI, or use the ``create`` subcommand. After creating a sync job, you can
+sync job, click the add button in the GUI, or use the ``create`` subcommand.
-either start it manually on the GUI or provide it with a schedule (see
+After creating a sync job, you can either start it manually from the GUI or
-:ref:`calendar-events`) to run regularly.
+provide it with a schedule (see :ref:`calendar-events`) to run regularly.
 
 .. code-block:: console
 
@@ -86,7 +86,7 @@ For setting up sync jobs, the configuring user needs the following permissions:
 
 If the ``remove-vanished`` option is set, ``Datastore.Prune`` is required on
 the local datastore as well. If the ``owner`` option is not set (defaulting to
-``backup@pam``) or set to something other than the configuring user,
+``root@pam``) or set to something other than the configuring user,
 ``Datastore.Modify`` is required as well.
 
 .. note:: A sync job can only sync backup groups that the configured remote's

docs/network-management.rst

@@ -1,3 +1,5 @@
+.. _sysadmin_network_configuration:
+
 Network Management
 ==================
 

docs/package-repositories.rst

@@ -26,11 +26,8 @@ update``.
 
 .. FIXME for 7.0: change security update suite to bullseye-security
 
-In addition, you need a package repository from Proxmox to get Proxmox Backup updates.
+In addition, you need a package repository from Proxmox to get Proxmox Backup
-
+updates.
-During the Proxmox Backup beta phase, only one repository (pbstest) will be
-available. Once released, an Enterprise repository for production use and a
-no-subscription repository will be provided.
 
 SecureApt
 ~~~~~~~~~
@@ -72,68 +69,63 @@ Here, the output should be:
 
  f3f6c5a3a67baf38ad178e5ff1ee270c /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg
 
-.. comment
+`Proxmox Backup`_ Enterprise Repository
-    `Proxmox Backup`_ Enterprise Repository
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    This will be the default, stable, and recommended repository. It is available for
+This will be the default, stable, and recommended repository. It is available for
-    all `Proxmox Backup`_ subscription users. It contains the most stable packages,
+all `Proxmox Backup`_ subscription users. It contains the most stable packages,
-    and is suitable for production use. The ``pbs-enterprise`` repository is
+and is suitable for production use. The ``pbs-enterprise`` repository is
-    enabled by default:
+enabled by default:
 
-    .. note:: During the Proxmox Backup beta phase only one repository (pbstest)
+.. code-block:: sources.list
-     will be available.
+  :caption: File: ``/etc/apt/sources.list.d/pbs-enterprise.list``
 
-    .. code-block:: sources.list
+  deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise
-      :caption: File: ``/etc/apt/sources.list.d/pbs-enterprise.list``
-
-      deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise
 
 
-    To never miss important security fixes, the superuser (``root@pam`` user) is
+To never miss important security fixes, the superuser (``root@pam`` user) is
-    notified via email about new packages as soon as they are available. The
+notified via email about new packages as soon as they are available. The
-    change-log and details of each package can be viewed in the GUI (if available).
+change-log and details of each package can be viewed in the GUI (if available).
 
-    Please note that you need a valid subscription key to access this
+Please note that you need a valid subscription key to access this
-    repository. More information regarding subscription levels and pricing can be
+repository. More information regarding subscription levels and pricing can be
-    found at https://www.proxmox.com/en/proxmox-backup/pricing.
+found at https://www.proxmox.com/en/proxmox-backup-server/pricing
 
-    .. note:: You can disable this repository by commenting out the above
+.. note:: You can disable this repository by commenting out the above line
-      line using a `#` (at the start of the line). This prevents error
+ using a `#` (at the start of the line). This prevents error messages if you do
-      messages if you do not have a subscription key. Please configure the
+ not have a subscription key. Please configure the ``pbs-no-subscription``
-      ``pbs-no-subscription`` repository in that case.
+ repository in that case.
 
 
-    `Proxmox Backup`_ No-Subscription Repository
+`Proxmox Backup`_ No-Subscription Repository
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    As the name suggests, you do not need a subscription key to access
+As the name suggests, you do not need a subscription key to access
-    this repository. It can be used for testing and non-production
+this repository. It can be used for testing and non-production
-    use. It is not recommended to use it on production servers, because these
+use. It is not recommended to use it on production servers, because these
-    packages are not always heavily tested and validated.
+packages are not always heavily tested and validated.
 
-    We recommend to configure this repository in ``/etc/apt/sources.list``.
+We recommend to configure this repository in ``/etc/apt/sources.list``.
 
-    .. code-block:: sources.list
+.. code-block:: sources.list
-      :caption: File: ``/etc/apt/sources.list``
+  :caption: File: ``/etc/apt/sources.list``
 
-      deb http://ftp.debian.org/debian buster main contrib
+  deb http://ftp.debian.org/debian buster main contrib
-      deb http://ftp.debian.org/debian buster-updates main contrib
+  deb http://ftp.debian.org/debian buster-updates main contrib
 
-      # PBS pbs-no-subscription repository provided by proxmox.com,
+  # PBS pbs-no-subscription repository provided by proxmox.com,
-      # NOT recommended for production use
+  # NOT recommended for production use
-      deb http://download.proxmox.com/debian/pbs buster pbs-no-subscription
+  deb http://download.proxmox.com/debian/pbs buster pbs-no-subscription
 
-      # security updates
+  # security updates
-      deb http://security.debian.org/debian-security buster/updates main contrib
+  deb http://security.debian.org/debian-security buster/updates main contrib
 
 
-`Proxmox Backup`_ Beta Repository
+`Proxmox Backup`_ Test Repository
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-During the public beta, there is a repository called ``pbstest``. This one
+This repository contains the latest packages and is heavily used by developers
-contains the latest packages and is heavily used by developers to test new
+to test new features.
-features.
 
 .. .. warning:: the ``pbstest`` repository should (as the name implies)
   only be used to test new features or bug fixes.
@@ -145,7 +137,3 @@ You can access this repository by adding the following line to
   :caption: sources.list entry for ``pbstest``
 
   deb http://download.proxmox.com/debian/pbs buster pbstest
-
-If you installed Proxmox Backup Server from the official beta ISO, you should
-have this repository already configured in
-``/etc/apt/sources.list.d/pbstest-beta.list``

docs/prune-simulator/documentation.html

@@ -0,0 +1,102 @@
+<!DOCTYPE html>
+<html>
+<head>
+<style>
+    /* similar to sphinx alabaster theme ones */
+    body {
+        max-width: 90ch;
+        margin-left: 2ch;
+        margin-right: 2ch;
+        line-height: 1.4em;
+        /* avoid the very high contrast of black on white, tone it down a bit */
+        color: #3E4349;
+        hyphens: auto;
+        text-align: left;
+        font-family: 'Open Sans', sans-serif;
+        font-size: 17px;
+    }
+    h1, h2, h3 {
+        font-family: Lato, sans-serif;
+        font-size: 150%;
+        line-height:1.2
+    }
+    tt, code {
+        background-color: #ecf0f3;
+        color: #222;
+    }
+    pre, tt, code {
+        font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace;
+        font-size: 0.9em;
+    }
+    div.note {
+        background-color: #EEE;
+        border: 1px solid #CCC;
+        margin: 10px 0;
+        padding: 0px 20px;
+    }
+    p.note-title {
+        font-weight: bolder;
+        padding: 0;
+        margin: 10px 0 0 0;
+    }
+    div.note > p.last {
+        margin: 5px 0 10px 0;
+    }
+</style>
+</head>
+<body>
+<p>A simulator to experiment with different backup schedules and prune
+options.</p>
+
+<h3>Schedule</h3>
+<p>Select weekdays with the combobox and input hour and minute
+specification separated by a colon, i.e. <code>HOUR:MINUTE</code>. Each of
+<code>HOUR</code> and <code>MINUTE</code> can be either a single value or
+one of the following:</p>
+<ul class="simple">
+<li>a comma-separated list: e.g., <code>01,02,03</code></li>
+<li>a range: e.g., <code>01..10</code></li>
+<li>a repetition: e.g, <code>05/10</code> (means starting at <code>5</code> every <code>10</code>)</li>
+<li>a combination of the above: e.g., <code>01,05..10,12/02</code></li>
+<li>a <code>*</code> for every possible value</li>
+</ul>
+
+<h3>Pruning</h3>
+<p>Prune lets you systematically delete older backups, retaining backups for
+the last given number of time intervals. The following retention options are
+available:</p>
+<dl class="docutils">
+<dt><code class="docutils literal notranslate"><span class="pre">keep-last</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> backup snapshots.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-hourly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> hours. If there is more than one
+backup for a single hour, only the latest is kept.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-daily</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> days. If there is more than one
+backup for a single day, only the latest is kept.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-weekly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> weeks. If there is more than one
+backup for a single week, only the latest is kept.
+<div class="last admonition note">
+<p class="note-title">Note:</p>
+<p class="last">Weeks start on Monday and end on Sunday. The software
+uses the <a class="reference external" href="https://en.wikipedia.org/wiki/ISO_week_date">ISO week date</a> system and handles weeks at
+the end of the year correctly.</p>
+</div>
+</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-monthly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> months. If there is more than one
+backup for a single month, only the latest is kept.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-yearly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> years. If there is more than one
+backup for a single year, only the latest is kept.</dd>
+</dl>
+<p>The retention options are processed in the order given above. Each option
+only covers backups within its time period. The next option does not take care
+of already covered backups. It will only consider older backups.</p>
+<p>For example, in a week covered by <code>keep-weekly</code>, one backup is
+kept while all others are removed; <code>keep-monthly</code> then does not
+consider backups from that week anymore, even if part of the week is part of
+an earlier month.</p>
+</body>
+</html>

docs/prune-simulator/index.html

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
+    <title>PBS Prune Simulator</title>
+
+    <link rel="stylesheet" type="text/css" href="extjs/theme-crisp/resources/theme-crisp-all.css">
+    <style>
+        .cal {
+            margin: 5px;
+        }
+        .cal-day {
+            vertical-align: top;
+            width: 150px;
+            height: 75px; /* this is like min-height when used in tables */
+            border: #939393 1px solid;
+            color: #454545;
+        }
+        .cal-day-date {
+            border-bottom: #444 1px solid;
+            color: #000;
+        }
+        .strikethrough {
+            text-decoration: line-through;
+        }
+        .black {
+            color: #000;
+        }
+        .sun {
+            background-color: #ededed;
+        }
+        .first-of-month {
+            border-right: dashed black 4px;
+        }
+    </style>
+
+    <script type="text/javascript" src="extjs/ext-all.js"></script>
+    <script type="text/javascript" src="prune-simulator.js"></script>
+</head>
+<body></body>
+</html>

docs/prune-simulator/prune-simulator.js

@@ -0,0 +1,783 @@
+// FIXME: HACK! Makes scrolling in number spinner work again. fixed in ExtJS >= 6.1
+if (Ext.isFirefox) {
+    Ext.$eventNameMap.DOMMouseScroll = 'DOMMouseScroll';
+}
+
+Ext.onReady(function() {
+    const NOW = new Date();
+    const COLORS = {
+	'keep-last': 'orange',
+	'keep-hourly': 'purple',
+	'keep-daily': 'yellow',
+	'keep-weekly': 'green',
+	'keep-monthly': 'blue',
+	'keep-yearly': 'red',
+	'all zero': 'white',
+    };
+    const TEXT_COLORS = {
+	'keep-last': 'black',
+	'keep-hourly': 'white',
+	'keep-daily': 'black',
+	'keep-weekly': 'white',
+	'keep-monthly': 'white',
+	'keep-yearly': 'white',
+	'all zero': 'black',
+    };
+
+    Ext.define('PBS.prunesimulator.Documentation', {
+	extend: 'Ext.Panel',
+	alias: 'widget.prunesimulatorDocumentation',
+
+	html: '<iframe style="width:100%;height:100%;border:0px;" src="./documentation.html"/>',
+    });
+
+    Ext.define('PBS.prunesimulator.CalendarEvent', {
+	extend: 'Ext.form.field.ComboBox',
+	alias: 'widget.prunesimulatorCalendarEvent',
+
+	editable: true,
+
+	displayField: 'text',
+	valueField: 'value',
+	queryMode: 'local',
+
+	store: {
+	    field: ['value', 'text'],
+	    data: [
+		{ value: '0/2:00', text: "Every two hours" },
+		{ value: '0/6:00', text: "Every six hours" },
+		{ value: '2,22:30', text: "At 02:30 and 22:30" },
+		{ value: '00:00', text: "At 00:00" },
+		{ value: '08..17:00/30', text: "From 08:00 to 17:30 every 30 minutes" },
+		{ value: 'HOUR:MINUTE', text: "Custom schedule" },
+	    ],
+	},
+
+	tpl: [
+	    '<ul class="x-list-plain"><tpl for=".">',
+	    '<li role="option" class="x-boundlist-item">{text}</li>',
+	    '</tpl></ul>',
+	],
+
+	displayTpl: [
+	    '<tpl for=".">',
+	    '{value}',
+	    '</tpl>',
+	],
+    });
+
+    Ext.define('PBS.prunesimulator.DayOfWeekSelector', {
+	extend: 'Ext.form.field.ComboBox',
+	alias: 'widget.prunesimulatorDayOfWeekSelector',
+
+	editable: false,
+
+	displayField: 'text',
+	valueField: 'value',
+	queryMode: 'local',
+
+	store: {
+	    field: ['value', 'text'],
+	    data: [
+		{ value: 'mon', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[1]) },
+		{ value: 'tue', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[2]) },
+		{ value: 'wed', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[3]) },
+		{ value: 'thu', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[4]) },
+		{ value: 'fri', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[5]) },
+		{ value: 'sat', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[6]) },
+		{ value: 'sun', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[0]) },
+	    ],
+	},
+    });
+
+    Ext.define('pbs-prune-list', {
+	extend: 'Ext.data.Model',
+	fields: [
+	    {
+		name: 'backuptime',
+		type: 'date',
+		dateFormat: 'timestamp',
+	    },
+	    {
+		name: 'mark',
+		type: 'string',
+	    },
+	    {
+		name: 'keepName',
+		type: 'string',
+	    },
+	],
+    });
+
+    Ext.define('PBS.prunesimulator.PruneList', {
+	extend: 'Ext.panel.Panel',
+	alias: 'widget.prunesimulatorPruneList',
+
+	initComponent: function() {
+	    let me = this;
+
+	    if (!me.store) {
+		throw "no store specified";
+	    }
+
+	    me.items = [
+		{
+		    xtype: 'grid',
+		    store: me.store,
+		    border: false,
+		    columns: [
+			{
+			    header: 'Backup Time',
+			    dataIndex: 'backuptime',
+			    renderer: function(value, metaData, record) {
+				let text = Ext.Date.format(value, 'Y-m-d H:i:s');
+				if (record.data.mark === 'keep') {
+				    if (me.useColors) {
+					let bgColor = COLORS[record.data.keepName];
+					let textColor = TEXT_COLORS[record.data.keepName];
+					return '<div style="background-color: ' + bgColor + '; ' +
+							    'color: ' + textColor + ';">' + text + '</div>';
+				    } else {
+					return text;
+				    }
+				} else {
+				    return '<div style="text-decoration: line-through;">' + text + '</div>';
+				}
+			    },
+			    flex: 1,
+			    sortable: false,
+			},
+			{
+			    header: 'Keep (reason)',
+			    dataIndex: 'mark',
+			    renderer: function(value, metaData, record) {
+				if (record.data.mark === 'keep') {
+				    if (record.data.keepCount) {
+					return 'keep (' + record.data.keepName +
+					       ': ' + record.data.keepCount + ')';
+				    } else {
+					return 'keep (' + record.data.keepName + ')';
+				    }
+				} else {
+				    return value;
+				}
+			    },
+			    width: 200,
+			    sortable: false,
+			},
+		    ],
+		},
+	    ];
+
+	    me.callParent();
+	},
+    });
+
+    Ext.define('PBS.prunesimulator.WeekTable', {
+	extend: 'Ext.panel.Panel',
+	alias: 'widget.prunesimulatorWeekTable',
+
+	reload: function() {
+	    let me = this;
+	    let backups = me.store.data.items;
+
+	    let html = '<table class="cal">';
+
+	    let now = new Date(NOW.getTime());
+	    let skip = 7 - parseInt(Ext.Date.format(now, 'N'), 10);
+	    let tableStartDate = Ext.Date.add(now, Ext.Date.DAY, skip);
+
+	    let bIndex = 0;
+
+	    for (let i = 0; bIndex < backups.length; i++) {
+		html += '<tr>';
+
+		for (let j = 0; j < 7; j++) {
+		    let date = Ext.Date.subtract(tableStartDate, Ext.Date.DAY, j + 7 * i);
+		    let currentDay = Ext.Date.format(date, 'd/m/Y');
+
+		    let dayOfWeekCls = Ext.Date.format(date, 'D').toLowerCase();
+		    let firstOfMonthCls = Ext.Date.format(date, 'd') === '01'
+		        ? 'first-of-month'
+		        : '';
+		    html += `<td class="cal-day ${dayOfWeekCls} ${firstOfMonthCls}">`;
+
+		    const isBackupOnDay = function(backup, day) {
+			return backup && Ext.Date.format(backup.data.backuptime, 'd/m/Y') === day;
+		    };
+
+		    let backup = backups[bIndex];
+
+		    html += '<table><tr>';
+		    html += `<th class="cal-day-date">${Ext.Date.format(date, 'D, d M Y')}</th>`;
+
+		    while (isBackupOnDay(backup, currentDay)) {
+			html += '<tr><td>';
+
+			let text = Ext.Date.format(backup.data.backuptime, 'H:i');
+			if (backup.data.mark === 'remove') {
+			    html += `<span class="strikethrough">${text}</span>`;
+			} else {
+			    if (backup.data.keepCount) {
+				text += ` (${backup.data.keepName} ${backup.data.keepCount})`;
+			    } else {
+				text += ` (${backup.data.keepName})`;
+			    }
+			    if (me.useColors) {
+				let bgColor = COLORS[backup.data.keepName];
+				let textColor = TEXT_COLORS[backup.data.keepName];
+				html += `<span style="background-color: ${bgColor};
+				    color: ${textColor};">${text}</span>`;
+			    } else {
+				html += `<span class="black">${text}</span>`;
+			    }
+			}
+			html += '</td></tr>';
+			backup = backups[++bIndex];
+		    }
+		    html += '</table>';
+		    html += '</div>';
+		    html += '</td>';
+		}
+
+		html += '</tr>';
+	    }
+
+	    me.setHtml(html);
+	},
+
+	initComponent: function() {
+	    let me = this;
+
+	    if (!me.store) {
+		throw "no store specified";
+	    }
+
+	    let reload = function() {
+		me.reload();
+	    };
+
+	    me.store.on("datachanged", reload);
+
+	    me.callParent();
+
+	    me.reload();
+	},
+    });
+
+    Ext.define('PBS.PruneSimulatorPanel', {
+	extend: 'Ext.panel.Panel',
+	alias: 'widget.prunesimulatorPanel',
+
+	viewModel: {
+	},
+
+	getValues: function() {
+	    let me = this;
+
+	    let values = {};
+
+	    Ext.Array.each(me.query('[isFormField]'), function(field) {
+		let data = field.getSubmitData();
+		Ext.Object.each(data, function(name, val) {
+		    values[name] = val;
+		});
+	    });
+
+	    return values;
+	},
+
+	controller: {
+	    xclass: 'Ext.app.ViewController',
+
+	    init: function(view) {
+		this.reloadFull(); // initial load
+		this.switchColor(true);
+	    },
+
+	    control: {
+		'field[fieldGroup=keep]': { change: 'reloadPrune' },
+	    },
+
+	    reloadFull: function() {
+		let me = this;
+		let view = me.getView();
+
+		let params = view.getValues();
+
+		let [hourSpec, minuteSpec] = params['schedule-time'].split(':');
+
+		if (!hourSpec || !minuteSpec) {
+		    Ext.Msg.alert('Error', 'Invalid schedule');
+		    return;
+		}
+
+		let matchTimeSpec = function(timeSpec, rangeMin, rangeMax) {
+		    let specValues = timeSpec.split(',');
+		    let matches = {};
+
+		    let assertValid = function(value) {
+			let num = Number(value);
+			if (isNaN(num)) {
+			    throw value + " is not an integer";
+			} else if (value < rangeMin || value > rangeMax) {
+			    throw "number '" + value + "' is not in the range '" + rangeMin + ".." + rangeMax + "'";
+			}
+			return num;
+		    };
+
+		    specValues.forEach(function(value) {
+			if (value.includes('..')) {
+			    let [start, end] = value.split('..');
+			    start = assertValid(start);
+			    end = assertValid(end);
+			    if (start > end) {
+				throw "interval start is bigger then interval end '" + start + " > " + end + "'";
+			    }
+			    for (let i = start; i <= end; i++) {
+				matches[i] = 1;
+			    }
+			} else if (value.includes('/')) {
+			    let [start, step] = value.split('/');
+			    start = assertValid(start);
+			    step = assertValid(step);
+			    for (let i = start; i <= rangeMax; i += step) {
+				matches[i] = 1;
+			    }
+			} else if (value === '*') {
+			    for (let i = rangeMin; i <= rangeMax; i++) {
+				matches[i] = 1;
+			    }
+			} else {
+			    value = assertValid(value);
+			    matches[value] = 1;
+			}
+		    });
+
+		    return Object.keys(matches);
+		};
+
+		let hours, minutes;
+
+		try {
+		    hours = matchTimeSpec(hourSpec, 0, 23);
+		    minutes = matchTimeSpec(minuteSpec, 0, 59);
+		} catch (err) {
+		    Ext.Msg.alert('Error', err);
+		    return;
+		}
+
+		let backups = me.populateFromSchedule(
+		    params['schedule-weekdays'],
+		    hours,
+		    minutes,
+		    params.numberOfWeeks,
+		);
+
+		me.pruneSelect(backups, params);
+
+		view.pruneStore.setData(backups);
+	    },
+
+	    reloadPrune: function() {
+		let me = this;
+		let view = me.getView();
+
+		let params = view.getValues();
+
+		let backups = [];
+		view.pruneStore.getData().items.forEach(function(item) {
+		    backups.push({
+			backuptime: item.data.backuptime,
+		    });
+		});
+
+		me.pruneSelect(backups, params);
+
+		view.pruneStore.setData(backups);
+	    },
+
+	    // backups are sorted descending by date
+	    populateFromSchedule: function(weekdays, hours, minutes, weekCount) {
+		let weekdayFlags = [
+		    weekdays.includes('sun'),
+		    weekdays.includes('mon'),
+		    weekdays.includes('tue'),
+		    weekdays.includes('wed'),
+		    weekdays.includes('thu'),
+		    weekdays.includes('fri'),
+		    weekdays.includes('sat'),
+		];
+
+		let todaysDate = new Date(NOW.getTime());
+
+		let timesOnSingleDay = [];
+
+		hours.forEach(function(hour) {
+		    minutes.forEach(function(minute) {
+			todaysDate.setHours(hour);
+			todaysDate.setMinutes(minute);
+			timesOnSingleDay.push(todaysDate.getTime());
+		    });
+		});
+
+		// ordering here and iterating backwards through days
+		// ensures that everything is ordered
+		timesOnSingleDay.sort(function(a, b) {
+		    return a < b;
+		});
+
+		let backups = [];
+
+		for (let i = 0; i < 7 * weekCount; i++) {
+		    let daysDate = Ext.Date.subtract(todaysDate, Ext.Date.DAY, i);
+		    let weekday = parseInt(Ext.Date.format(daysDate, 'w'), 10);
+		    if (weekdayFlags[weekday]) {
+			timesOnSingleDay.forEach(function(time) {
+			    backups.push({
+				backuptime: Ext.Date.subtract(new Date(time), Ext.Date.DAY, i),
+			    });
+			});
+		    }
+		}
+
+		return backups;
+	    },
+
+	    pruneMark: function(backups, keepCount, keepName, idFunc) {
+		if (!keepCount) {
+		    return;
+		}
+
+		let alreadyIncluded = {};
+		let newlyIncluded = {};
+		let newlyIncludedCount = 0;
+
+		let finished = false;
+
+		backups.forEach(function(backup) {
+		    let mark = backup.mark;
+		    let id = idFunc(backup);
+
+		    if (finished || alreadyIncluded[id]) {
+			return;
+		    }
+
+		    if (mark) {
+			if (mark === 'keep') {
+			    alreadyIncluded[id] = true;
+			}
+			return;
+		    }
+
+		    if (!newlyIncluded[id]) {
+			if (newlyIncludedCount >= keepCount) {
+			    finished = true;
+			    return;
+			}
+			newlyIncluded[id] = true;
+			newlyIncludedCount++;
+			backup.mark = 'keep';
+			backup.keepName = keepName;
+			backup.keepCount = newlyIncludedCount;
+		    } else {
+			backup.mark = 'remove';
+		    }
+		});
+	    },
+
+	    // backups need to be sorted descending by date
+	    pruneSelect: function(backups, keepParams) {
+		let me = this;
+
+		if (Number(keepParams['keep-last']) +
+		    Number(keepParams['keep-hourly']) +
+		    Number(keepParams['keep-daily']) +
+		    Number(keepParams['keep-weekly']) +
+		    Number(keepParams['keep-monthly']) +
+		    Number(keepParams['keep-yearly']) === 0) {
+		    backups.forEach(function(backup) {
+			backup.mark = 'keep';
+			backup.keepName = 'keep-all';
+		    });
+
+		    return;
+		}
+
+		me.pruneMark(backups, keepParams['keep-last'], 'keep-last', function(backup) {
+		    return backup.backuptime;
+		});
+		me.pruneMark(backups, keepParams['keep-hourly'], 'keep-hourly', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'H/d/m/Y');
+		});
+		me.pruneMark(backups, keepParams['keep-daily'], 'keep-daily', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'd/m/Y');
+		});
+		me.pruneMark(backups, keepParams['keep-weekly'], 'keep-weekly', function(backup) {
+		    // ISO-8601 week and week-based year
+		    return Ext.Date.format(backup.backuptime, 'W/o');
+		});
+		me.pruneMark(backups, keepParams['keep-monthly'], 'keep-monthly', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'm/Y');
+		});
+		me.pruneMark(backups, keepParams['keep-yearly'], 'keep-yearly', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'Y');
+		});
+
+		backups.forEach(function(backup) {
+		    backup.mark = backup.mark || 'remove';
+		});
+	    },
+
+	    toggleColors: function(checkbox, checked) {
+		this.switchColor(checked);
+	    },
+
+	    switchColor: function(useColors) {
+		let me = this;
+		let view = me.getView();
+
+		const getStyle = name =>
+		    `background-color: ${COLORS[name]}; color: ${TEXT_COLORS[name]};`;
+
+		for (const field of view.query('[isFormField]')) {
+		    if (field.fieldGroup !== 'keep') {
+			continue;
+		    }
+		    if (useColors) {
+		    field.setFieldStyle(getStyle(field.name));
+		    } else {
+			field.setFieldStyle('background-color: white; color: #444;');
+		    }
+		}
+
+		me.lookup('weekTable').useColors = useColors;
+		me.lookup('pruneList').useColors = useColors;
+
+		me.reloadPrune();
+	    },
+	},
+
+	keepItems: [
+	    {
+		xtype: 'numberfield',
+		name: 'keep-last',
+		allowBlank: true,
+		fieldLabel: 'keep-last',
+		minValue: 0,
+		value: 4,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-hourly',
+		allowBlank: true,
+		fieldLabel: 'keep-hourly',
+		minValue: 0,
+		value: 0,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-daily',
+		allowBlank: true,
+		fieldLabel: 'keep-daily',
+		minValue: 0,
+		value: 5,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-weekly',
+		allowBlank: true,
+		fieldLabel: 'keep-weekly',
+		minValue: 0,
+		value: 2,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-monthly',
+		allowBlank: true,
+		fieldLabel: 'keep-monthly',
+		minValue: 0,
+		value: 0,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-yearly',
+		allowBlank: true,
+		fieldLabel: 'keep-yearly',
+		minValue: 0,
+		value: 0,
+		fieldGroup: 'keep',
+	    },
+	],
+
+	initComponent: function() {
+	    var me = this;
+
+	    me.pruneStore = Ext.create('Ext.data.Store', {
+		model: 'pbs-prune-list',
+		sorters: { property: 'backuptime', direction: 'DESC' },
+	    });
+
+	    me.items = [
+		{
+		    xtype: 'panel',
+		    layout: {
+			type: 'hbox',
+			align: 'stretch',
+		    },
+		    border: false,
+		    items: [
+			{
+			    title: 'View',
+			    layout: 'anchor',
+			    flex: 1,
+			    border: false,
+			    bodyPadding: 10,
+			    items: [
+				{
+				    xtype: 'checkbox',
+				    name: 'showCalendar',
+				    reference: 'showCalendar',
+				    fieldLabel: 'Show Calendar:',
+				    checked: true,
+				},
+				{
+				    xtype: 'checkbox',
+				    name: 'showColors',
+				    reference: 'showColors',
+				    fieldLabel: 'Show Colors:',
+				    checked: true,
+				    handler: 'toggleColors',
+				},
+			    ],
+			},
+			{ xtype: "panel", width: 1, border: 1 },
+			{
+			    xtype: 'form',
+			    layout: 'anchor',
+			    flex: 1,
+			    border: false,
+			    title: 'Simulated Backup Schedule',
+			    defaults: {
+				labelWidth: 120,
+			    },
+			    bodyPadding: 10,
+			    items: [
+				{
+				    xtype: 'prunesimulatorDayOfWeekSelector',
+				    name: 'schedule-weekdays',
+				    fieldLabel: 'Day of week',
+				    value: ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'],
+				    allowBlank: false,
+				    multiSelect: true,
+				    padding: '0 0 0 10',
+				},
+				{
+				    xtype: 'prunesimulatorCalendarEvent',
+				    name: 'schedule-time',
+				    allowBlank: false,
+				    value: '0/6:00',
+				    fieldLabel: 'Backup schedule',
+				    padding: '0 0 0 10',
+				},
+				{
+				    xtype: 'numberfield',
+				    name: 'numberOfWeeks',
+				    allowBlank: false,
+				    fieldLabel: 'Number of weeks',
+				    minValue: 1,
+				    value: 15,
+				    maxValue: 260, // five years
+				    padding: '0 0 0 10',
+				},
+				{
+				    xtype: 'button',
+				    name: 'schedule-button',
+				    text: 'Update Schedule',
+				    formBind: true,
+				    handler: 'reloadFull',
+				},
+			    ],
+			},
+		    ],
+		},
+		{
+		    xtype: 'panel',
+		    layout: {
+			type: 'hbox',
+			align: 'stretch',
+		    },
+		    flex: 1,
+		    border: false,
+		    items: [
+			{
+			    layout: 'anchor',
+			    title: 'Prune Options',
+			    border: false,
+			    bodyPadding: 10,
+			    scrollable: true,
+			    items: me.keepItems,
+			    flex: 1,
+			},
+			{ xtype: "panel", width: 1, border: 1 },
+			{
+			    layout: 'fit',
+			    title: 'Backups',
+			    border: false,
+			    xtype: 'prunesimulatorPruneList',
+			    store: me.pruneStore,
+			    reference: 'pruneList',
+			    flex: 1,
+			},
+		    ],
+		},
+		{
+		    layout: 'anchor',
+		    title: 'Calendar',
+		    autoScroll: true,
+		    flex: 2,
+		    xtype: 'prunesimulatorWeekTable',
+		    reference: 'weekTable',
+		    store: me.pruneStore,
+		    bind: {
+			hidden: '{!showCalendar.checked}',
+		    },
+		},
+	    ];
+
+	    me.callParent();
+	},
+    });
+
+    Ext.create('Ext.container.Viewport', {
+	layout: 'border',
+	renderTo: Ext.getBody(),
+	items: [
+	    {
+		xtype: 'prunesimulatorPanel',
+		title: 'Proxmox Backup Server - Prune Simulator',
+		region: 'west',
+		layout: {
+		    type: 'vbox',
+		    align: 'stretch',
+		    pack: 'start',
+		},
+		flex: 3,
+		maxWidth: 1090,
+	    },
+	    {
+		xtype: 'prunesimulatorDocumentation',
+		title: 'Usage',
+		border: false,
+		flex: 2,
+		region: 'center',
+	    },
+	],
+    });
+});
+

docs/storage.rst

@@ -1,6 +1,8 @@
 Storage
 =======
 
+.. _storage_disk_management:
+
 Disk Management
 ---------------
 
@@ -57,7 +59,7 @@ create a datastore at the location ``/mnt/datastore/store1``:
 You can also create a ``zpool`` with various raid levels from **Administration
 -> Disks -> Zpool** in the web interface, or by using ``zpool create``. The command
 below creates a mirrored ``zpool`` using two disks (``sdb`` & ``sdc``) and
-mounts it on the root directory (default):
+mounts it under ``/mnt/datastore/zpool1``:
 
 .. code-block:: console
 
@@ -85,7 +87,7 @@ display S.M.A.R.T. attributes from the web interface or by using the command:
 
 .. _datastore_intro:
 
-:term:`DataStore`
+:term:`Datastore`
 -----------------
 
 A datastore refers to a location at which backups are stored. The current
@@ -107,7 +109,7 @@ is stored in the file ``/etc/proxmox-backup/datastore.cfg``.
 Datastore Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-.. image:: images/screenshots/pbs-gui-datastore.png
+.. image:: images/screenshots/pbs-gui-datastore-content.png
   :align: right
   :alt: Datastore Overview
 
@@ -121,14 +123,17 @@ number of backups to keep in that store. :ref:`backup-pruning` and
 periodically based on a configured schedule (see :ref:`calendar-events`) per datastore.
 
 
+.. _storage_datastore_create:
+
 Creating a Datastore
 ^^^^^^^^^^^^^^^^^^^^
 .. image:: images/screenshots/pbs-gui-datastore-create-general.png
   :align: right
   :alt: Create a datastore
 
-You can create a new datastore from the web GUI, by navigating to **Datastore** in
+You can create a new datastore from the web interface, by clicking **Add
-the menu tree and clicking **Create**. Here:
+Datastore** in the side menu, under the **Datastore** section. In the setup
+window:
 
 * *Name* refers to the name of the datastore
 * *Backing Path* is the path to the directory upon which you want to create the
@@ -136,7 +141,9 @@ the menu tree and clicking **Create**. Here:
 * *GC Schedule* refers to the time and intervals at which garbage collection
   runs
 * *Prune Schedule* refers to the frequency at which pruning takes place
-* *Prune Options* set the amount of backups which you would like to keep (see :ref:`backup-pruning`).
+* *Prune Options* set the amount of backups which you would like to keep (see
+  :ref:`backup-pruning`).
+* *Comment* can be used to add some contextual information to the datastore.
 
 Alternatively you can create a new datastore from the command line. The
 following command creates a new datastore called ``store1`` on :file:`/backup/disk1/store1`

docs/sysadmin.rst

@@ -1,3 +1,5 @@
+.. _sysadmin_host_administration:
+
 Host System Administration
 ==========================
 

docs/user-management.rst

@@ -41,11 +41,12 @@ users:
   :alt: Add a new user
 
 The superuser has full administration rights on everything, so you
-normally want to add other users with less privileges. You can create a new
+normally want to add other users with less privileges. You can add a new
-user with the ``user create`` subcommand or through the web interface, under
+user with the ``user create`` subcommand or through the web
-**Configuration -> User Management**. The ``create`` subcommand lets you specify
+interface, under the **User Management** tab of **Configuration -> Access
-many options like ``--email`` or ``--password``. You can update or change any
+Control**. The ``create`` subcommand lets you specify many options like
-user properties using the ``update`` subcommand later (**Edit** in the GUI):
+``--email`` or ``--password``. You can update or change any user properties
+using the ``update`` subcommand later (**Edit** in the GUI):
 
 
 .. code-block:: console
@@ -90,6 +91,10 @@ Or completely remove the user with:
 API Tokens
 ----------
 
+.. image:: images/screenshots/pbs-gui-apitoken-overview.png
+  :align: right
+  :alt: API Token Overview
+
 Any authenticated user can generate API tokens which can in turn be used to
 configure various clients, instead of directly providing the username and
 password.
@@ -104,6 +109,10 @@ the realm and a tokenname (``user@realm!tokenname``), and a secret value. Both
 need to be provided to the client in place of the user ID (``user@realm``) and
 the user password, respectively.
 
+.. image:: images/screenshots/pbs-gui-apitoken-secret-value.png
+  :align: right
+  :alt: API secret value
+
 The API token is passed from the client to the server by setting the
 ``Authorization`` HTTP header with method ``PBSAPIToken`` to the value
 ``TOKENID:TOKENSECRET``.
@@ -184,7 +193,7 @@ following roles exist:
 **RemoteSyncOperator**
   Is allowed to read data from a remote.
 
-.. image:: images/screenshots/pbs-gui-permissions-add.png
+.. image:: images/screenshots/pbs-gui-user-management-add-user.png
   :align: right
   :alt: Add permissions for user
 
@@ -221,11 +230,11 @@ You can list the ACLs of each user/token using the following command:
 .. code-block:: console
 
    # proxmox-backup-manager acl list
-   ┌──────────┬──────────────────┬───────────┬────────────────┐
+   ┌──────────┬───────────────────┬───────────┬────────────────┐
-   │ ugid     │ path             │ propagate │ roleid         │
+   │ ugid     │ path              │ propagate │ roleid         │
-   ╞══════════╪══════════════════╪═══════════╪════════════════╡
+   ╞══════════╪═══════════════════╪═══════════╪════════════════╡
-   │ john@pbs │ /datastore/disk1 │         1 │ DatastoreAdmin │
+   │ john@pbs │ /datastore/store1 │         1 │ DatastoreAdmin │
-   └──────────┴──────────────────┴───────────┴────────────────┘
+   └──────────┴───────────────────┴───────────┴────────────────┘
 
 A single user/token can be assigned multiple permission sets for different datastores.
 
@@ -258,7 +267,7 @@ you can use the ``proxmox-backup-manager user permission`` command:
 
 .. code-block:: console
 
-  # proxmox-backup-manager user permissions john@pbs -- path /datastore/store1
+  # proxmox-backup-manager user permissions john@pbs --path /datastore/store1
   Privileges with (*) have the propagate flag set
   
   Path: /datastore/store1
@@ -267,9 +276,10 @@ you can use the ``proxmox-backup-manager user permission`` command:
   - Datastore.Modify (*)
   - Datastore.Prune (*)
   - Datastore.Read (*)
+  - Datastore.Verify (*)
   
   # proxmox-backup-manager acl update /datastore/store1 DatastoreBackup --auth-id 'john@pbs!client1'
-  # proxmox-backup-manager user permissions 'john@pbs!test' -- path /datastore/store1
+  # proxmox-backup-manager user permissions 'john@pbs!client1' --path /datastore/store1
   Privileges with (*) have the propagate flag set
   
   Path: /datastore/store1

etc/Makefile

@@ -9,7 +9,7 @@ DYNAMIC_UNITS := \
 	proxmox-backup.service \
 	proxmox-backup-proxy.service
 
-all: $(UNITS) $(DYNAMIC_UNITS) pbstest-beta.list
+all: $(UNITS) $(DYNAMIC_UNITS) pbs-enterprise.list
 
 clean:
 	rm -f $(DYNAMIC_UNITS)

etc/pbs-enterprise.list

@@ -0,0 +1 @@
+deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise

etc/pbstest-beta.list

@@ -1 +0,0 @@
-deb http://download.proxmox.com/debian/pbs buster pbstest

src/api2/access.rs

@@ -12,7 +12,6 @@ use proxmox::{http_err, list_subdirs_api_method};
 use crate::tools::ticket::{self, Empty, Ticket};
 use crate::auth_helpers::*;
 use crate::api2::types::*;
-use crate::tools::{FileLogOptions, FileLogger};
 
 use crate::config::acl as acl_config;
 use crate::config::acl::{PRIVILEGES, PRIV_SYS_AUDIT, PRIV_PERMISSIONS_MODIFY};
@@ -144,20 +143,13 @@ fn create_ticket(
     port: Option<u16>,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
-    let logger_options = FileLogOptions {
-        append: true,
-        prefix_time: true,
-        ..Default::default()
-    };
-    let mut auth_log = FileLogger::new("/var/log/proxmox-backup/api/auth.log", logger_options)?;
-
     match authenticate_user(&username, &password, path, privs, port) {
         Ok(true) => {
             let ticket = Ticket::new("PBS", &username)?.sign(private_auth_key(), None)?;
 
             let token = assemble_csrf_prevention_token(csrf_secret(), &username);
 
-            auth_log.log(format!("successful auth for user '{}'", username));
+            crate::server::rest::auth_logger()?.log(format!("successful auth for user '{}'", username));
 
             Ok(json!({
                 "username": username,
@@ -180,7 +172,7 @@ fn create_ticket(
                 username,
                 err.to_string()
             );
-            auth_log.log(&msg);
+            crate::server::rest::auth_logger()?.log(&msg);
             log::error!("{}", msg);
 
             Err(http_err!(UNAUTHORIZED, "permission check failed."))

src/api2/access/user.rs

@@ -75,7 +75,7 @@ pub struct UserWithTokens {
     pub lastname: Option<String>,
     #[serde(skip_serializing_if="Option::is_none")]
     pub email: Option<String>,
-    #[serde(skip_serializing_if="Vec::is_empty")]
+    #[serde(skip_serializing_if="Vec::is_empty", default)]
     pub tokens: Vec<user::ApiToken>,
 }
 

src/api2/admin/datastore.rs

@@ -187,7 +187,13 @@ fn list_groups(
         let group = info.backup_dir.group();
 
         let list_all = (user_privs & PRIV_DATASTORE_AUDIT) != 0;
-        let owner = datastore.get_owner(group)?;
+        let owner = match datastore.get_owner(group) {
+            Ok(auth_id) => auth_id,
+            Err(err) => {
+                println!("Failed to get owner of group '{}' - {}", group, err);
+                continue;
+            },
+        };
         if !list_all && check_backup_owner(&owner, &auth_id).is_err() {
             continue;
         }
@@ -369,7 +375,13 @@ pub fn list_snapshots (
         }
 
         let list_all = (user_privs & PRIV_DATASTORE_AUDIT) != 0;
-        let owner = datastore.get_owner(group)?;
+        let owner = match datastore.get_owner(group) {
+            Ok(auth_id) => auth_id,
+            Err(err) => {
+                println!("Failed to get owner of group '{}' - {}", group, err);
+                continue;
+            },
+        };
 
         if !list_all && check_backup_owner(&owner, &auth_id).is_err() {
             continue;
@@ -636,7 +648,7 @@ pub fn verify(
                 verify_all_backups(datastore, worker.clone(), worker.upid(), owner, None)?
             };
             if failed_dirs.len() > 0 {
-                worker.log("Failed to verify following snapshots:");
+                worker.log("Failed to verify following snapshots/groups:");
                 for dir in failed_dirs {
                     worker.log(format!("\t{}", dir));
                 }
@@ -902,15 +914,7 @@ pub fn garbage_collection_status(
         type: Array,
         items: {
             description: "Datastore name and description.",
-            properties: {
+            type: DataStoreListItem,
-                store: {
-                    schema: DATASTORE_SCHEMA,
-                },
-                comment: {
-                    optional: true,
-                    schema: SINGLE_LINE_COMMENT_SCHEMA,
-                },
-            },
         },
     },
     access: {
@@ -922,7 +926,7 @@ fn get_datastore_list(
     _param: Value,
     _info: &ApiMethod,
     rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
+) -> Result<Vec<DataStoreListItem>, Error> {
 
     let (config, _digest) = datastore::config()?;
 
@@ -935,11 +939,12 @@ fn get_datastore_list(
         let user_privs = user_info.lookup_privs(&auth_id, &["datastore", &store]);
         let allowed = (user_privs & (PRIV_DATASTORE_AUDIT| PRIV_DATASTORE_BACKUP)) != 0;
         if allowed {
-            let mut entry = json!({ "store": store });
+            list.push(
-            if let Some(comment) = data["comment"].as_str() {
+                DataStoreListItem {
-                entry["comment"] = comment.into();
+                    store: store.clone(),
-            }
+                    comment: data["comment"].as_str().map(String::from),
-            list.push(entry);
+                }
+            );
         }
     }
 

src/api2/admin/verify.rs

@@ -2,11 +2,16 @@ use anyhow::{format_err, Error};
 
 use proxmox::api::router::SubdirMap;
 use proxmox::{list_subdirs_api_method, sortable};
-use proxmox::api::{api, ApiMethod, Router, RpcEnvironment};
+use proxmox::api::{api, ApiMethod, Permission, Router, RpcEnvironment};
 
 use crate::api2::types::*;
 use crate::server::do_verification_job;
 use crate::server::jobstate::{Job, JobState};
+use crate::config::acl::{
+    PRIV_DATASTORE_AUDIT,
+    PRIV_DATASTORE_VERIFY,
+};
+use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::verify;
 use crate::config::verify::{VerificationJobConfig, VerificationJobStatus};
 use serde_json::Value;
@@ -23,10 +28,14 @@ use crate::server::UPID;
         },
     },
     returns: {
-        description: "List configured jobs and their status.",
+        description: "List configured jobs and their status (filtered by access)",
         type: Array,
         items: { type: verify::VerificationJobStatus },
     },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Audit or Datastore.Verify on datastore.",
+    },
 )]
 /// List all verification jobs
 pub fn list_verification_jobs(
@@ -34,6 +43,10 @@ pub fn list_verification_jobs(
     _param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<VerificationJobStatus>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_VERIFY;
 
     let (config, digest) = verify::config()?;
 
@@ -41,6 +54,11 @@ pub fn list_verification_jobs(
         .convert_to_typed_array("verification")?
         .into_iter()
         .filter(|job: &VerificationJobStatus| {
+            let privs = user_info.lookup_privs(&auth_id, &["datastore", &job.store]);
+            if privs & required_privs == 0 {
+                return false;
+            }
+
             if let Some(store) = &store {
                 &job.store == store
             } else {
@@ -90,7 +108,11 @@ pub fn list_verification_jobs(
                 schema: JOB_ID_SCHEMA,
             }
         }
-    }
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Verify on job's datastore.",
+    },
 )]
 /// Runs a verification job manually.
 fn run_verification_job(
@@ -98,10 +120,13 @@ fn run_verification_job(
     _info: &ApiMethod,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<String, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
     let (config, _digest) = verify::config()?;
     let verification_job: VerificationJobConfig = config.lookup("verification", &id)?;
 
-    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    user_info.check_privs(&auth_id, &["datastore", &verification_job.store], PRIV_DATASTORE_VERIFY, true)?;
 
     let job = Job::new("verificationjob", &id)?;
 

src/api2/config/datastore.rs

@@ -5,6 +5,7 @@ use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, Router, RpcEnvironment, Permission};
+use proxmox::api::schema::parse_property_string;
 use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
@@ -68,6 +69,14 @@ pub fn list_datastores(
                 optional: true,
                 schema: SINGLE_LINE_COMMENT_SCHEMA,
             },
+            "notify-user": {
+                optional: true,
+                type: Userid,
+            },
+            "notify": {
+                optional: true,
+                schema: DATASTORE_NOTIFY_STRING_SCHEMA,
+            },
             "gc-schedule": {
                 optional: true,
                 schema: GC_SCHEDULE_SCHEMA,
@@ -187,6 +196,12 @@ pub enum DeletableProperty {
     keep_monthly,
     /// Delete the keep-yearly property
     keep_yearly,
+    /// Delete the verify-new property
+    verify_new,
+    /// Delete the notify-user property
+    notify_user,
+    /// Delete the notify property
+    notify,
 }
 
 #[api(
@@ -200,6 +215,14 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: SINGLE_LINE_COMMENT_SCHEMA,
             },
+            "notify-user": {
+                optional: true,
+                type: Userid,
+            },
+            "notify": {
+                optional: true,
+                schema: DATASTORE_NOTIFY_STRING_SCHEMA,
+            },
             "gc-schedule": {
                 optional: true,
                 schema: GC_SCHEDULE_SCHEMA,
@@ -232,6 +255,12 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_YEARLY,
             },
+            "verify-new": {
+                description: "If enabled, all new backups will be verified right after completion.",
+                type: bool,
+                optional: true,
+                default: false,
+            },
             delete: {
                 description: "List of properties to delete.",
                 type: Array,
@@ -262,6 +291,9 @@ pub fn update_datastore(
     keep_weekly: Option<u64>,
     keep_monthly: Option<u64>,
     keep_yearly: Option<u64>,
+    verify_new: Option<bool>,
+    notify: Option<String>,
+    notify_user: Option<Userid>,
     delete: Option<Vec<DeletableProperty>>,
     digest: Option<String>,
 ) -> Result<(), Error> {
@@ -290,6 +322,9 @@ pub fn update_datastore(
                 DeletableProperty::keep_weekly => { data.keep_weekly = None; },
                 DeletableProperty::keep_monthly => { data.keep_monthly = None; },
                 DeletableProperty::keep_yearly => { data.keep_yearly = None; },
+                DeletableProperty::verify_new => { data.verify_new = None; },
+                DeletableProperty::notify => { data.notify = None; },
+                DeletableProperty::notify_user => { data.notify_user = None; },
             }
         }
     }
@@ -322,6 +357,19 @@ pub fn update_datastore(
     if keep_monthly.is_some() { data.keep_monthly = keep_monthly; }
     if keep_yearly.is_some() { data.keep_yearly = keep_yearly; }
 
+    if let Some(notify_str) = notify {
+        let value = parse_property_string(&notify_str, &DatastoreNotify::API_SCHEMA)?;
+        let notify: DatastoreNotify = serde_json::from_value(value)?;
+        if let  DatastoreNotify { gc: None, verify: None, sync: None } = notify {
+            data.notify = None;
+        } else {
+            data.notify = Some(notify_str);
+        }
+    }
+    if verify_new.is_some() { data.verify_new = verify_new; }
+
+    if notify_user.is_some() { data.notify_user = notify_user; }
+
     config.set_data(&name, "datastore", &data)?;
 
     datastore::save_config(&config)?;

src/api2/config/remote.rs

@@ -1,11 +1,13 @@
-use anyhow::{bail, Error};
+use anyhow::{bail, format_err, Error};
 use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
+use proxmox::http_err;
 use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
+use crate::client::{HttpClient, HttpClientOptions};
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::remote;
 use crate::config::acl::{PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY};
@@ -76,7 +78,7 @@ pub fn list_remotes(
                 optional: true,
                 default: 8007,
             },
-            userid: {
+            "auth-id": {
                 type: Authid,
             },
             password: {
@@ -176,7 +178,7 @@ pub enum DeletableProperty {
                 type: u16,
                 optional: true,
             },
-            userid: {
+            "auth-id": {
                 optional: true,
                 type: Authid,
             },
@@ -212,7 +214,7 @@ pub fn update_remote(
     comment: Option<String>,
     host: Option<String>,
     port: Option<u16>,
-    userid: Option<Authid>,
+    auth_id: Option<Authid>,
     password: Option<String>,
     fingerprint: Option<String>,
     delete: Option<Vec<DeletableProperty>>,
@@ -250,7 +252,7 @@ pub fn update_remote(
     }
     if let Some(host) = host { data.host = host; }
     if port.is_some() { data.port = port; }
-    if let Some(userid) = userid { data.userid = userid; }
+    if let Some(auth_id) = auth_id { data.auth_id = auth_id; }
     if let Some(password) = password { data.password = password; }
 
     if let Some(fingerprint) = fingerprint { data.fingerprint = Some(fingerprint); }
@@ -282,6 +284,17 @@ pub fn update_remote(
 /// Remove a remote from the configuration file.
 pub fn delete_remote(name: String, digest: Option<String>) -> Result<(), Error> {
 
+    use crate::config::sync::{self, SyncJobConfig};
+
+    let (sync_jobs, _) = sync::config()?;
+
+    let job_list: Vec<SyncJobConfig>  = sync_jobs.convert_to_typed_array("sync")?;
+    for job in job_list {
+        if job.remote == name {
+            bail!("remote '{}' is used by sync job '{}' (datastore '{}')", name, job.id, job.store);
+        }
+    }
+
     let _lock = open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = remote::config()?;
@@ -301,10 +314,83 @@ pub fn delete_remote(name: String, digest: Option<String>) -> Result<(), Error>
     Ok(())
 }
 
+/// Helper to get client for remote.cfg entry
+pub async fn remote_client(remote: remote::Remote) -> Result<HttpClient, Error> {
+    let options = HttpClientOptions::new()
+        .password(Some(remote.password.clone()))
+        .fingerprint(remote.fingerprint.clone());
+
+    let client = HttpClient::new(
+        &remote.host,
+        remote.port.unwrap_or(8007),
+        &remote.auth_id,
+        options)?;
+    let _auth_info = client.login() // make sure we can auth
+        .await
+        .map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?;
+
+    Ok(client)
+}
+
+
+#[api(
+    input: {
+        properties: {
+            name: {
+                schema: REMOTE_ID_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["remote", "{name}"], PRIV_REMOTE_AUDIT, false),
+    },
+    returns: {
+        description: "List the accessible datastores.",
+        type: Array,
+        items: {
+            description: "Datastore name and description.",
+            type: DataStoreListItem,
+        },
+    },
+)]
+/// List datastores of a remote.cfg entry
+pub async fn scan_remote_datastores(name: String) -> Result<Vec<DataStoreListItem>, Error> {
+    let (remote_config, _digest) = remote::config()?;
+    let remote: remote::Remote = remote_config.lookup("remote", &name)?;
+
+    let map_remote_err = |api_err| {
+        http_err!(INTERNAL_SERVER_ERROR,
+                  "failed to scan remote '{}' - {}",
+                  &name,
+                  api_err)
+    };
+
+    let client = remote_client(remote)
+        .await
+        .map_err(map_remote_err)?;
+    let api_res = client
+        .get("api2/json/admin/datastore", None)
+        .await
+        .map_err(map_remote_err)?;
+    let parse_res = match api_res.get("data") {
+        Some(data) => serde_json::from_value::<Vec<DataStoreListItem>>(data.to_owned()),
+        None => bail!("remote {} did not return any datastore list data", &name),
+    };
+
+    match parse_res {
+        Ok(parsed) => Ok(parsed),
+        Err(_) => bail!("Failed to parse remote scan api result."),
+    }
+}
+
+const SCAN_ROUTER: Router = Router::new()
+    .get(&API_METHOD_SCAN_REMOTE_DATASTORES);
+
 const ITEM_ROUTER: Router = Router::new()
     .get(&API_METHOD_READ_REMOTE)
     .put(&API_METHOD_UPDATE_REMOTE)
-    .delete(&API_METHOD_DELETE_REMOTE);
+    .delete(&API_METHOD_DELETE_REMOTE)
+    .subdirs(&[("scan", &SCAN_ROUTER)]);
 
 pub const ROUTER: Router = Router::new()
     .get(&API_METHOD_LIST_REMOTES)

src/api2/config/sync.rs

@@ -58,7 +58,7 @@ pub fn check_sync_job_modify_access(
                     && owner.user() == auth_id.user())
         },
         // default sync owner
-        None => auth_id == Authid::backup_auth_id(),
+        None => auth_id == Authid::root_auth_id(),
     };
 
     // same permission as changing ownership after syncing
@@ -511,7 +511,7 @@ acl:1:/remote/remote1/remotestore1:write@pbs:RemoteSyncOperator
     job.owner = Some(read_auth_id.clone());
     assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
 
-    // also not to the default 'backup@pam'
+    // also not to the default 'root@pam'
     job.owner = None;
     assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
 

src/api2/config/verify.rs

@@ -9,10 +9,11 @@ use crate::api2::types::*;
 
 use crate::config::acl::{
     PRIV_DATASTORE_AUDIT,
-    PRIV_DATASTORE_BACKUP,
     PRIV_DATASTORE_VERIFY,
 };
 
+use crate::config::cached_user_info::CachedUserInfo;
+
 use crate::config::verify::{self, VerificationJobConfig};
 
 #[api(
@@ -25,10 +26,8 @@ use crate::config::verify::{self, VerificationJobConfig};
         items: { type: verify::VerificationJobConfig },
     },
     access: {
-        permission: &Permission::Privilege(
+        permission: &Permission::Anybody,
-            &["datastore", "{store}"],
+        description: "Requires Datastore.Audit or Datastore.Verify on datastore.",
-            PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_BACKUP | PRIV_DATASTORE_VERIFY,
-            true),
     },
 )]
 /// List all verification jobs
@@ -36,11 +35,22 @@ pub fn list_verification_jobs(
     _param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<VerificationJobConfig>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_VERIFY;
 
     let (config, digest) = verify::config()?;
 
     let list = config.convert_to_typed_array("verification")?;
 
+    let list = list.into_iter()
+        .filter(|job: &VerificationJobConfig| {
+            let privs = user_info.lookup_privs(&auth_id, &["datastore", &job.store]);
+
+            privs & required_privs != 00
+        }).collect();
+
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
     Ok(list)
@@ -76,19 +86,24 @@ pub fn list_verification_jobs(
         }
     },
     access: {
-        permission: &Permission::Privilege(
+        permission: &Permission::Anybody,
-            &["datastore", "{store}"],
+        description: "Requires Datastore.Verify on job's datastore.",
-            PRIV_DATASTORE_VERIFY,
-            true),
     },
 )]
 /// Create a new verification job.
-pub fn create_verification_job(param: Value) -> Result<(), Error> {
+pub fn create_verification_job(
-
+    param: Value,
-    let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+    rpcenv: &mut dyn RpcEnvironment
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let verification_job: verify::VerificationJobConfig = serde_json::from_value(param.clone())?;
 
+    user_info.check_privs(&auth_id, &["datastore", &verification_job.store], PRIV_DATASTORE_VERIFY, false)?;
+
+    let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
     let (mut config, _digest) = verify::config()?;
 
     if let Some(_) = config.sections.get(&verification_job.id) {
@@ -117,10 +132,8 @@ pub fn create_verification_job(param: Value) -> Result<(), Error> {
         type: verify::VerificationJobConfig,
     },
     access: {
-        permission: &Permission::Privilege(
+        permission: &Permission::Anybody,
-            &["datastore", "{store}"],
+        description: "Requires Datastore.Audit or Datastore.Verify on job's datastore.",
-            PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_BACKUP | PRIV_DATASTORE_VERIFY,
-            true),
     },
 )]
 /// Read a verification job configuration.
@@ -128,9 +141,16 @@ pub fn read_verification_job(
     id: String,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<VerificationJobConfig, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
     let (config, digest) = verify::config()?;
 
-    let verification_job = config.lookup("verification", &id)?;
+    let verification_job: verify::VerificationJobConfig = config.lookup("verification", &id)?;
+
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_VERIFY;
+    user_info.check_privs(&auth_id, &["datastore", &verification_job.store], required_privs, true)?;
+
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
     Ok(verification_job)
@@ -193,10 +213,8 @@ pub enum DeletableProperty {
         },
     },
     access: {
-        permission: &Permission::Privilege(
+        permission: &Permission::Anybody,
-            &["datastore", "{store}"],
+        description: "Requires Datastore.Verify on job's datastore.",
-            PRIV_DATASTORE_VERIFY,
-            true),
     },
 )]
 /// Update verification job config.
@@ -209,7 +227,10 @@ pub fn update_verification_job(
     schedule: Option<String>,
     delete: Option<Vec<DeletableProperty>>,
     digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
@@ -223,7 +244,10 @@ pub fn update_verification_job(
 
     let mut data: verify::VerificationJobConfig = config.lookup("verification", &id)?;
 
-     if let Some(delete) = delete {
+    // check existing store
+    user_info.check_privs(&auth_id, &["datastore", &data.store], PRIV_DATASTORE_VERIFY, true)?;
+
+    if let Some(delete) = delete {
         for delete_prop in delete {
             match delete_prop {
                 DeletableProperty::IgnoreVerified => { data.ignore_verified = None; },
@@ -243,7 +267,12 @@ pub fn update_verification_job(
         }
     }
 
-    if let Some(store) = store { data.store = store; }
+    if let Some(store) = store {
+        // check new store
+        user_info.check_privs(&auth_id, &["datastore", &store], PRIV_DATASTORE_VERIFY, true)?;
+        data.store = store;
+    }
+
 
     if ignore_verified.is_some() { data.ignore_verified = ignore_verified; }
     if outdated_after.is_some() { data.outdated_after = outdated_after; }
@@ -270,19 +299,26 @@ pub fn update_verification_job(
         },
     },
     access: {
-        permission: &Permission::Privilege(
+        permission: &Permission::Anybody,
-            &["datastore", "{store}"],
+        description: "Requires Datastore.Verify on job's datastore.",
-            PRIV_DATASTORE_VERIFY,
-            true),
     },
 )]
 /// Remove a verification job configuration
-pub fn delete_verification_job(id: String, digest: Option<String>) -> Result<(), Error> {
+pub fn delete_verification_job(
+    id: String,
+    digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = verify::config()?;
 
+    let job: verify::VerificationJobConfig = config.lookup("verification", &id)?;
+    user_info.check_privs(&auth_id, &["datastore", &job.store], PRIV_DATASTORE_VERIFY, true)?;
+
     if let Some(ref digest) = digest {
         let digest = proxmox::tools::hex_to_digest(digest)?;
         crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;

src/api2/node/apt.rs

@@ -1,12 +1,13 @@
 use anyhow::{Error, bail, format_err};
 use serde_json::{json, Value};
+use std::collections::HashMap;
 
 use proxmox::list_subdirs_api_method;
 use proxmox::api::{api, RpcEnvironment, RpcEnvironmentType, Permission};
 use proxmox::api::router::{Router, SubdirMap};
 
 use crate::server::WorkerTask;
-use crate::tools::{apt, http};
+use crate::tools::{apt, http, subscription};
 
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
 use crate::api2::types::{Authid, APTUpdateInfo, NODE_SCHEMA, UPID_SCHEMA};
@@ -202,9 +203,34 @@ fn apt_get_changelog(
     let changelog_url = &pkg_info[0].change_log_url;
     // FIXME: use 'apt-get changelog' for proxmox packages as well, once repo supports it
     if changelog_url.starts_with("http://download.proxmox.com/") {
-        let changelog = crate::tools::runtime::block_on(http::get_string(changelog_url))
+        let changelog = crate::tools::runtime::block_on(http::get_string(changelog_url, None))
             .map_err(|err| format_err!("Error downloading changelog from '{}': {}", changelog_url, err))?;
         return Ok(json!(changelog));
+
+    } else if changelog_url.starts_with("https://enterprise.proxmox.com/") {
+        let sub = match subscription::read_subscription()? {
+            Some(sub) => sub,
+            None => bail!("cannot retrieve changelog from enterprise repo: no subscription info found")
+        };
+        let (key, id) = match sub.key {
+            Some(key) => {
+                match sub.serverid {
+                    Some(id) => (key, id),
+                    None =>
+                        bail!("cannot retrieve changelog from enterprise repo: no server id found")
+                }
+            },
+            None => bail!("cannot retrieve changelog from enterprise repo: no subscription key found")
+        };
+
+        let mut auth_header = HashMap::new();
+        auth_header.insert("Authorization".to_owned(),
+            format!("Basic {}", base64::encode(format!("{}:{}", key, id))));
+
+        let changelog = crate::tools::runtime::block_on(http::get_string(changelog_url, Some(&auth_header)))
+            .map_err(|err| format_err!("Error downloading changelog from '{}': {}", changelog_url, err))?;
+        return Ok(json!(changelog));
+
     } else {
         let mut command = std::process::Command::new("apt-get");
         command.arg("changelog");
@@ -215,12 +241,113 @@ fn apt_get_changelog(
     }
 }
 
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+        },
+    },
+    returns: {
+        description: "List of more relevant packages.",
+        type: Array,
+        items: {
+            type: APTUpdateInfo,
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// Get package information for important Proxmox Backup Server packages.
+pub fn get_versions() -> Result<Value, Error> {
+    const PACKAGES: &[&str] = &[
+        "ifupdown2",
+        "libjs-extjs",
+        "proxmox-backup",
+        "proxmox-backup-docs",
+        "proxmox-backup-client",
+        "proxmox-backup-server",
+        "proxmox-mini-journalreader",
+        "proxmox-widget-toolkit",
+        "pve-xtermjs",
+        "smartmontools",
+        "zfsutils-linux",
+    ];
+
+    fn unknown_package(package: String) -> APTUpdateInfo {
+        APTUpdateInfo {
+            package,
+            title: "unknown".into(),
+            arch: "unknown".into(),
+            description: "unknown".into(),
+            version: "unknown".into(),
+            old_version: "unknown".into(),
+            origin: "unknown".into(),
+            priority: "unknown".into(),
+            section: "unknown".into(),
+            change_log_url: "unknown".into(),
+        }
+    }
+
+    let is_kernel = |name: &str| name.starts_with("pve-kernel-");
+
+    let mut packages: Vec<APTUpdateInfo> = Vec::new();
+    let pbs_packages = apt::list_installed_apt_packages(
+        |filter| {
+            filter.installed_version == Some(filter.active_version)
+                && (is_kernel(filter.package) || PACKAGES.contains(&filter.package))
+        },
+        None,
+    );
+    if let Some(proxmox_backup) = pbs_packages.iter().find(|pkg| pkg.package == "proxmox-backup") {
+        packages.push(proxmox_backup.to_owned());
+    } else {
+        packages.push(unknown_package("proxmox-backup".into()));
+    }
+
+    if let Some(pkg) = pbs_packages.iter().find(|pkg| pkg.package == "proxmox-backup-server") {
+        packages.push(pkg.to_owned());
+    }
+
+    let mut kernel_pkgs: Vec<APTUpdateInfo> = pbs_packages
+        .iter()
+        .filter(|pkg| is_kernel(&pkg.package))
+        .cloned()
+        .collect();
+    // make sure the cache mutex gets dropped before the next call to list_installed_apt_packages
+    {
+        let cache = apt_pkg_native::Cache::get_singleton();
+        kernel_pkgs.sort_by(|left, right| {
+            cache
+                .compare_versions(&left.old_version, &right.old_version)
+                .reverse()
+        });
+    }
+    packages.append(&mut kernel_pkgs);
+
+    // add entry for all packages we're interested in, even if not installed
+    for pkg in PACKAGES.iter() {
+        if pkg == &"proxmox-backup" || pkg == &"proxmox-backup-server" {
+            continue;
+        }
+        match pbs_packages.iter().find(|item| &item.package == pkg) {
+            Some(apt_pkg) => packages.push(apt_pkg.to_owned()),
+            None => packages.push(unknown_package(pkg.to_string())),
+        }
+    }
+
+    Ok(json!(packages))
+}
+
 const SUBDIRS: SubdirMap = &[
     ("changelog", &Router::new().get(&API_METHOD_APT_GET_CHANGELOG)),
     ("update", &Router::new()
         .get(&API_METHOD_APT_UPDATE_AVAILABLE)
         .post(&API_METHOD_APT_UPDATE_DATABASE)
     ),
+    ("versions", &Router::new().get(&API_METHOD_GET_VERSIONS)),
 ];
 
 pub const ROUTER: Router = Router::new()

src/api2/node/disks/directory.rs

@@ -142,6 +142,18 @@ pub fn create_datastore_disk(
         bail!("disk '{}' is already in use.", disk);
     }
 
+    let mount_point = format!("/mnt/datastore/{}", &name);
+
+    // check if the default path does exist already and bail if it does
+    let default_path = std::path::PathBuf::from(&mount_point);
+
+    match std::fs::metadata(&default_path) {
+        Err(_) => {}, // path does not exist
+        Ok(_) => {
+            bail!("path {:?} already exists", default_path);
+        }
+    }
+
     let upid_str = WorkerTask::new_thread(
         "dircreate", Some(name.clone()), auth_id, to_stdout, move |worker|
         {
@@ -160,7 +172,7 @@ pub fn create_datastore_disk(
             let uuid = get_fs_uuid(&partition)?;
             let uuid_path = format!("/dev/disk/by-uuid/{}", uuid);
 
-            let (mount_unit_name, mount_point) = create_datastore_mount_unit(&name, filesystem, &uuid_path)?;
+            let mount_unit_name = create_datastore_mount_unit(&name, &mount_point, filesystem, &uuid_path)?;
 
             systemd::reload_daemon()?;
             systemd::enable_unit(&mount_unit_name)?;
@@ -243,11 +255,11 @@ pub const ROUTER: Router = Router::new()
 
 fn create_datastore_mount_unit(
     datastore_name: &str,
+    mount_point: &str,
     fs_type: FileSystemType,
     what: &str,
-) -> Result<(String, String), Error> {
+) -> Result<String, Error> {
 
-    let mount_point = format!("/mnt/datastore/{}", datastore_name);
     let mut mount_unit_name = systemd::escape_unit(&mount_point, true);
     mount_unit_name.push_str(".mount");
 
@@ -265,7 +277,7 @@ fn create_datastore_mount_unit(
 
     let mount = SystemdMountSection {
         What: what.to_string(),
-        Where: mount_point.clone(),
+        Where: mount_point.to_string(),
         Type: Some(fs_type.to_string()),
         Options: Some(String::from("defaults")),
         ..Default::default()
@@ -278,5 +290,5 @@ fn create_datastore_mount_unit(
 
     systemd::config::save_systemd_mount(&mount_unit_path, &config)?;
 
-    Ok((mount_unit_name, mount_point))
+    Ok(mount_unit_name)
 }

src/api2/node/disks/zfs.rs

@@ -243,7 +243,7 @@ pub fn zpool_details(
         permission: &Permission::Privilege(&["system", "disks"], PRIV_SYS_MODIFY, false),
     },
 )]
-/// Create a new ZFS pool.
+/// Create a new ZFS pool. Will be mounted under '/mnt/datastore/<name>'.
 pub fn create_zpool(
     name: String,
     devices: String,
@@ -303,10 +303,11 @@ pub fn create_zpool(
         bail!("{:?} needs at least {} disks.", raidlevel, min_disks);
     }
 
+    let mount_point = format!("/mnt/datastore/{}", &name);
+
     // check if the default path does exist already and bail if it does
-    // otherwise we get an error on mounting
+    // otherwise 'zpool create' aborts after partitioning, but before creating the pool
-    let mut default_path = std::path::PathBuf::from("/");
+    let default_path = std::path::PathBuf::from(&mount_point);
-    default_path.push(&name);
 
     match std::fs::metadata(&default_path) {
         Err(_) => {}, // path does not exist
@@ -322,7 +323,7 @@ pub fn create_zpool(
 
 
             let mut command = std::process::Command::new("zpool");
-            command.args(&["create", "-o", &format!("ashift={}", ashift), &name]);
+            command.args(&["create", "-o", &format!("ashift={}", ashift), "-m", &mount_point, &name]);
 
             match raidlevel {
                 ZfsRaidLevel::Single => {
@@ -371,7 +372,6 @@ pub fn create_zpool(
             }
 
             if add_datastore {
-                let mount_point = format!("/{}", name);
                 crate::api2::config::datastore::create_datastore(json!({ "name": name, "path": mount_point }))?
             }
 

src/api2/node/tasks.rs

@@ -1,7 +1,7 @@
 use std::fs::File;
 use std::io::{BufRead, BufReader};
 
-use anyhow::{Error};
+use anyhow::{bail, Error};
 use serde_json::{json, Value};
 
 use proxmox::api::{api, Router, RpcEnvironment, Permission};
@@ -9,19 +9,117 @@ use proxmox::api::router::SubdirMap;
 use proxmox::{identity, list_subdirs_api_method, sortable};
 
 use crate::tools;
+
 use crate::api2::types::*;
+use crate::api2::pull::check_pull_privs;
+
 use crate::server::{self, UPID, TaskState, TaskListInfoIterator};
-use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
+use crate::config::acl::{
+    PRIV_DATASTORE_MODIFY,
+    PRIV_DATASTORE_VERIFY,
+    PRIV_SYS_AUDIT,
+    PRIV_SYS_MODIFY,
+};
 use crate::config::cached_user_info::CachedUserInfo;
 
+// matches respective job execution privileges
+fn check_job_privs(auth_id: &Authid, user_info: &CachedUserInfo, upid: &UPID) -> Result<(), Error> {
+    match (upid.worker_type.as_str(), &upid.worker_id) {
+        ("verificationjob", Some(workerid)) => {
+            if let Some(captures) = VERIFICATION_JOB_WORKER_ID_REGEX.captures(&workerid) {
+                if let Some(store) = captures.get(1) {
+                    return user_info.check_privs(&auth_id,
+                                                 &["datastore", store.as_str()],
+                                                 PRIV_DATASTORE_VERIFY,
+                                                 true);
+                }
+            }
+        },
+        ("syncjob", Some(workerid)) => {
+            if let Some(captures) = SYNC_JOB_WORKER_ID_REGEX.captures(&workerid) {
+                let remote = captures.get(1);
+                let remote_store = captures.get(2);
+                let local_store = captures.get(3);
+
+                if let (Some(remote), Some(remote_store), Some(local_store)) =
+                    (remote, remote_store, local_store) {
+
+                    return check_pull_privs(&auth_id,
+                                            local_store.as_str(),
+                                            remote.as_str(),
+                                            remote_store.as_str(),
+                                            false);
+                }
+            }
+        },
+        ("garbage_collection", Some(workerid)) => {
+            return user_info.check_privs(&auth_id,
+                                         &["datastore", &workerid],
+                                         PRIV_DATASTORE_MODIFY,
+                                         true)
+        },
+        ("prune", Some(workerid)) => {
+            return user_info.check_privs(&auth_id,
+                                         &["datastore",
+                                         &workerid],
+                                         PRIV_DATASTORE_MODIFY,
+                                         true);
+        },
+        _ => bail!("not a scheduled job task"),
+    };
+
+    bail!("not a scheduled job task");
+}
+
+// get the store out of the worker_id
+fn check_job_store(upid: &UPID, store: &str) -> bool {
+    match (upid.worker_type.as_str(), &upid.worker_id) {
+        (workertype, Some(workerid)) if workertype.starts_with("verif") => {
+            if let Some(captures) = VERIFICATION_JOB_WORKER_ID_REGEX.captures(&workerid) {
+                if let Some(jobstore) = captures.get(1) {
+                    return store == jobstore.as_str();
+                }
+            } else {
+                return workerid == store;
+            }
+        }
+        ("syncjob", Some(workerid)) => {
+            if let Some(captures) = SYNC_JOB_WORKER_ID_REGEX.captures(&workerid) {
+                if let Some(local_store) = captures.get(3) {
+                    return store == local_store.as_str();
+                }
+            }
+        }
+        ("prune", Some(workerid))
+        | ("backup", Some(workerid))
+        | ("garbage_collection", Some(workerid)) => {
+            return workerid == store || workerid.starts_with(&format!("{}:", store));
+        }
+        _ => {}
+    };
+
+    false
+}
+
 fn check_task_access(auth_id: &Authid, upid: &UPID) -> Result<(), Error> {
     let task_auth_id = &upid.auth_id;
     if auth_id == task_auth_id
         || (task_auth_id.is_token() && &Authid::from(task_auth_id.user().clone()) == auth_id) {
+        // task owner can always read
         Ok(())
     } else {
         let user_info = CachedUserInfo::new()?;
-        user_info.check_privs(auth_id, &["system", "tasks"], PRIV_SYS_AUDIT, false)
+
+        let task_privs = user_info.lookup_privs(auth_id, &["system", "tasks"]);
+        if task_privs & PRIV_SYS_AUDIT != 0 {
+            // allowed to read all tasks in general
+            Ok(())
+        } else if check_job_privs(&auth_id, &user_info, upid).is_ok() {
+            // job which the user/token could have configured/manually executed
+            Ok(())
+        } else {
+            bail!("task access not allowed");
+        }
     }
 }
 
@@ -387,21 +485,8 @@ pub fn list_tasks(
         }
 
         if let Some(store) = store {
-            // Note: useful to select all tasks spawned by proxmox-backup-client
+            if !check_job_store(&info.upid, store) {
-            let worker_id = match &info.upid.worker_id {
+                return None;
-                Some(w) => w,
-                None => return None, // skip
-            };
-
-            if info.upid.worker_type == "backup" || info.upid.worker_type == "restore" ||
-                info.upid.worker_type == "prune"
-            {
-                let prefix = format!("{}:", store);
-                if !worker_id.starts_with(&prefix) { return None; }
-            } else if info.upid.worker_type == "garbage_collection" {
-                if worker_id != store { return None; }
-            } else {
-                return None; // skip
             }
         }
 

src/api2/pull.rs

@@ -9,7 +9,7 @@ use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
 
 use crate::server::{WorkerTask, jobstate::Job};
 use crate::backup::DataStore;
-use crate::client::{HttpClient, HttpClientOptions, BackupRepository, pull::pull_store};
+use crate::client::{HttpClient, BackupRepository, pull::pull_store};
 use crate::api2::types::*;
 use crate::config::{
     remote,
@@ -50,17 +50,9 @@ pub async fn get_pull_parameters(
     let (remote_config, _digest) = remote::config()?;
     let remote: remote::Remote = remote_config.lookup("remote", remote)?;
 
-    let options = HttpClientOptions::new()
+    let src_repo = BackupRepository::new(Some(remote.auth_id.clone()), Some(remote.host.clone()), remote.port, remote_store.to_string());
-        .password(Some(remote.password.clone()))
-        .fingerprint(remote.fingerprint.clone());
-
-    let src_repo = BackupRepository::new(Some(remote.userid.clone()), Some(remote.host.clone()), remote.port, remote_store.to_string());
-
-    let client = HttpClient::new(&src_repo.host(), src_repo.port(), &src_repo.auth_id(), options)?;
-    let _auth_info = client.login() // make sure we can auth
-        .await
-        .map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?;
 
+    let client = crate::api2::config::remote::remote_client(remote).await?;
 
     Ok((client, src_repo, tgt_store))
 }
@@ -72,14 +64,18 @@ pub fn do_sync_job(
     schedule: Option<String>,
 ) -> Result<String, Error> {
 
-    let job_id = job.jobname().to_string();
+    let job_id = format!("{}:{}:{}:{}",
+                         sync_job.remote,
+                         sync_job.remote_store,
+                         sync_job.store,
+                         job.jobname());
     let worker_type = job.jobtype().to_string();
 
-    let email = crate::server::lookup_user_email(auth_id.user());
+    let (email, notify) = crate::server::lookup_datastore_notify_settings(&sync_job.store);
 
     let upid_str = WorkerTask::spawn(
         &worker_type,
-        Some(job.jobname().to_string()),
+        Some(job_id.clone()),
         auth_id.clone(),
         false,
         move |worker| async move {
@@ -92,7 +88,7 @@ pub fn do_sync_job(
             let worker_future = async move {
 
                 let delete = sync_job.remove_vanished.unwrap_or(true);
-                let sync_owner = sync_job.owner.unwrap_or(Authid::backup_auth_id().clone());
+                let sync_owner = sync_job.owner.unwrap_or(Authid::root_auth_id().clone());
                 let (client, src_repo, tgt_store) = get_pull_parameters(&sync_job.store, &sync_job.remote, &sync_job.remote_store).await?;
 
                 worker.log(format!("Starting datastore sync job '{}'", job_id));
@@ -126,7 +122,7 @@ pub fn do_sync_job(
             }
 
             if let Some(email) = email {
-                if let Err(err) = crate::server::send_sync_status(&email, &sync_job2, &result) {
+                if let Err(err) = crate::server::send_sync_status(&email, notify, &sync_job2, &result) {
                     eprintln!("send sync notification failed: {}", err);
                 }
             }

src/api2/status.rs

@@ -103,6 +103,7 @@ fn datastore_status(
             "total": status.total,
             "used": status.used,
             "avail": status.avail,
+            "gc-status": datastore.last_gc_status(),
         });
 
         let rrd_dir = format!("datastore/{}", store);
@@ -152,6 +153,8 @@ fn datastore_status(
                     }
                 }
 
+                entry["history-start"] = start.into();
+                entry["history-delta"] = reso.into();
                 entry["history"] = history.into();
 
                 // we skip the calculation for datastores with not enough data

src/api2/types/mod.rs

@@ -59,6 +59,11 @@ const_regex!{
     /// any identifier command line tools work with.
     pub PROXMOX_SAFE_ID_REGEX = concat!(r"^", PROXMOX_SAFE_ID_REGEX_STR!(), r"$");
 
+    /// Regex for verification jobs 'DATASTORE:ACTUAL_JOB_ID'
+    pub VERIFICATION_JOB_WORKER_ID_REGEX = concat!(r"^(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):");
+    /// Regex for sync jobs 'REMOTE:REMOTE_DATASTORE:LOCAL_DATASTORE:ACTUAL_JOB_ID'
+    pub SYNC_JOB_WORKER_ID_REGEX = concat!(r"^(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):");
+
     pub SINGLE_LINE_COMMENT_REGEX = r"^[[:^cntrl:]]*$";
 
     pub HOSTNAME_REGEX = r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9])?)$";
@@ -370,6 +375,25 @@ pub const BLOCKDEVICE_NAME_SCHEMA: Schema = StringSchema::new("Block device name
 
 // Complex type definitions
 
+#[api(
+    properties: {
+        store: {
+            schema: DATASTORE_SCHEMA,
+        },
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+    },
+)]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all="kebab-case")]
+/// Basic information about a datastore.
+pub struct DataStoreListItem {
+    pub store: String,
+    pub comment: Option<String>,
+}
+
 #[api(
     properties: {
         "backup-type": {
@@ -1129,7 +1153,7 @@ pub enum RRDTimeFrameResolution {
 }
 
 #[api()]
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "PascalCase")]
 /// Describes a package for which an update is available.
 pub struct APTUpdateInfo {
@@ -1154,3 +1178,48 @@ pub struct APTUpdateInfo {
     /// URL under which the package's changelog can be retrieved
     pub change_log_url: String,
 }
+
+#[api()]
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// When do we send notifications
+pub enum Notify {
+    /// Never send notification
+    Never,
+    /// Send notifications for failed and sucessful jobs
+    Always,
+    /// Send notifications for failed jobs only
+    Error,
+}
+
+#[api(
+    properties: {
+        gc: {
+            type: Notify,
+            optional: true,
+        },
+        verify: {
+            type: Notify,
+            optional: true,
+        },
+        sync: {
+            type: Notify,
+            optional: true,
+        },
+    },
+)]
+#[derive(Debug, Serialize, Deserialize)]
+/// Datastore notify settings
+pub struct DatastoreNotify {
+    /// Garbage collection settings
+    pub gc: Option<Notify>,
+    /// Verify job setting
+    pub verify: Option<Notify>,
+    /// Sync job setting
+    pub sync: Option<Notify>,
+}
+
+pub const DATASTORE_NOTIFY_STRING_SCHEMA: Schema = StringSchema::new(
+    "Datastore notification setting")
+    .format(&ApiStringFormat::PropertyString(&DatastoreNotify::API_SCHEMA))
+    .schema();

src/api2/types/userid.rs

@@ -450,11 +450,6 @@ impl Userid {
         &self.data
     }
 
-    /// Get the "backup@pam" user id.
-    pub fn backup_userid() -> &'static Self {
-        &*BACKUP_USERID
-    }
-
     /// Get the "root@pam" user id.
     pub fn root_userid() -> &'static Self {
         &*ROOT_USERID
@@ -462,7 +457,6 @@ impl Userid {
 }
 
 lazy_static! {
-    pub static ref BACKUP_USERID: Userid = Userid::new("backup@pam".to_string(), 6);
     pub static ref ROOT_USERID: Userid = Userid::new("root@pam".to_string(), 4);
 }
 
@@ -596,11 +590,6 @@ impl Authid {
         }
     }
 
-    /// Get the "backup@pam" auth id.
-    pub fn backup_auth_id() -> &'static Self {
-        &*BACKUP_AUTHID
-    }
-
     /// Get the "root@pam" auth id.
     pub fn root_auth_id() -> &'static Self {
         &*ROOT_AUTHID
@@ -608,7 +597,6 @@ impl Authid {
 }
 
 lazy_static! {
-    pub static ref BACKUP_AUTHID: Authid = Authid::from(Userid::new("backup@pam".to_string(), 6));
     pub static ref ROOT_AUTHID: Authid = Authid::from(Userid::new("root@pam".to_string(), 4));
 }
 

src/backup/verify.rs

@@ -498,28 +498,38 @@ pub fn verify_all_backups(
 ) -> Result<Vec<String>, Error> {
     let mut errors = Vec::new();
 
+    task_log!(worker, "verify datastore {}", datastore.name());
+
     if let Some(owner) = &owner {
-        task_log!(
+        task_log!(worker, "limiting to backups owned by {}", owner);
-            worker,
-            "verify datastore {} - limiting to backups owned by {}",
-            datastore.name(),
-            owner
-        );
     }
 
     let filter_by_owner = |group: &BackupGroup| {
-        if let Some(owner) = &owner {
+        match (datastore.get_owner(group), &owner) {
-            match datastore.get_owner(group) {
+            (Ok(ref group_owner), Some(owner)) => {
-                Ok(ref group_owner) => {
+                group_owner == owner
-                    group_owner == owner
+                    || (group_owner.is_token()
-                        || (group_owner.is_token()
+                        && !owner.is_token()
-                            && !owner.is_token()
+                        && group_owner.user() == owner.user())
-                            && group_owner.user() == owner.user())
+            },
-                },
+            (Ok(_), None) => true,
-                Err(_) => false,
+            (Err(err), Some(_)) => {
-            }
+                // intentionally not in task log
-        } else {
+                // the task user might not be allowed to see this group!
-            true
+                println!("Failed to get owner of group '{}' - {}", group, err);
+                false
+            },
+            (Err(err), None) => {
+                // we don't filter by owner, but we want to log the error
+                task_log!(
+                    worker,
+                    "Failed to get owner of group '{} - {}",
+                    group,
+                    err,
+                );
+                errors.push(group.to_string());
+                true
+            },
         }
     };
 
@@ -532,8 +542,7 @@ pub fn verify_all_backups(
         Err(err) => {
             task_log!(
                 worker,
-                "verify datastore {} - unable to list backups: {}",
+                "unable to list backups: {}",
-                datastore.name(),
                 err,
             );
             return Ok(errors);
@@ -553,7 +562,7 @@ pub fn verify_all_backups(
     // start with 64 chunks since we assume there are few corrupt ones
     let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
 
-    task_log!(worker, "verify datastore {} ({} snapshots)", datastore.name(), snapshot_count);
+    task_log!(worker, "found {} snapshots", snapshot_count);
 
     let mut done = 0;
     for group in list {

src/bin/proxmox-backup-api.rs

@@ -76,6 +76,7 @@ async fn run() -> Result<(), Error> {
                 })
             )
         },
+        "proxmox-backup.service",
     );
 
     server::write_pid(buildcfg::PROXMOX_BACKUP_API_PID_FN)?;

src/bin/proxmox-backup-client.rs

@@ -32,11 +32,11 @@ use proxmox::{
 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
 
 use proxmox_backup::tools;
+use proxmox_backup::api2::access::user::UserWithTokens;
 use proxmox_backup::api2::types::*;
 use proxmox_backup::api2::version;
 use proxmox_backup::client::*;
 use proxmox_backup::pxar::catalog::*;
-use proxmox_backup::config::user::complete_userid;
 use proxmox_backup::backup::{
     archive_type,
     decrypt_key,
@@ -193,8 +193,12 @@ pub fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<
     result
 }
 
-fn connect(server: &str, port: u16, auth_id: &Authid) -> Result<HttpClient, Error> {
+fn connect(repo: &BackupRepository) -> Result<HttpClient, Error> {
+    connect_do(repo.host(), repo.port(), repo.auth_id())
+        .map_err(|err| format_err!("error building client for repository {} - {}", repo, err))
+}
 
+fn connect_do(server: &str, port: u16, auth_id: &Authid) -> Result<HttpClient, Error> {
     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
 
     use std::env::VarError::*;
@@ -366,7 +370,7 @@ async fn list_backup_groups(param: Value) -> Result<Value, Error> {
 
     let repo = extract_repository_from_value(&param)?;
 
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
 
     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 
@@ -435,7 +439,7 @@ async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Erro
 
     let repo = extract_repository_from_value(&param)?;
 
-    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let mut client = connect(&repo)?;
 
     param.as_object_mut().unwrap().remove("repository");
 
@@ -478,7 +482,7 @@ async fn list_snapshots(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
 
     let group: Option<BackupGroup> = if let Some(path) = param["group"].as_str() {
         Some(path.parse()?)
@@ -543,7 +547,7 @@ async fn forget_snapshots(param: Value) -> Result<Value, Error> {
     let path = tools::required_string_param(&param, "snapshot")?;
     let snapshot: BackupDir = path.parse()?;
 
-    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let mut client = connect(&repo)?;
 
     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 
@@ -573,7 +577,7 @@ async fn api_login(param: Value) -> Result<Value, Error> {
 
     let repo = extract_repository_from_value(&param)?;
 
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
     client.login().await?;
 
     record_repository(&repo);
@@ -630,7 +634,7 @@ async fn api_version(param: Value) -> Result<(), Error> {
 
     let repo = extract_repository_from_value(&param);
     if let Ok(repo) = repo {
-        let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+        let client = connect(&repo)?;
 
         match client.get("api2/json/version", None).await {
             Ok(mut result) => version_info["server"] = result["data"].take(),
@@ -680,7 +684,7 @@ async fn list_snapshot_files(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
 
     let path = format!("api2/json/admin/datastore/{}/files", repo.store());
 
@@ -724,7 +728,7 @@ async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let mut client = connect(&repo)?;
 
     let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
 
@@ -1036,7 +1040,7 @@ async fn create_backup(
 
     let backup_time = backup_time_opt.unwrap_or_else(|| epoch_i64());
 
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
     record_repository(&repo);
 
     println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time)?);
@@ -1339,7 +1343,7 @@ async fn restore(param: Value) -> Result<Value, Error> {
 
     let archive_name = tools::required_string_param(&param, "archive-name")?;
 
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
 
     record_repository(&repo);
 
@@ -1512,7 +1516,7 @@ async fn upload_log(param: Value) -> Result<Value, Error> {
     let snapshot = tools::required_string_param(&param, "snapshot")?;
     let snapshot: BackupDir = snapshot.parse()?;
 
-    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let mut client = connect(&repo)?;
 
     let (keydata, crypt_mode) = keyfile_parameters(&param)?;
 
@@ -1583,7 +1587,7 @@ fn prune<'a>(
 async fn prune_async(mut param: Value) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
 
-    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let mut client = connect(&repo)?;
 
     let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
 
@@ -1669,7 +1673,7 @@ async fn status(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
 
     let path = format!("api2/json/admin/datastore/{}/status", repo.store());
 
@@ -1904,6 +1908,33 @@ fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<Stri
     result
 }
 
+fn complete_auth_id(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
+    proxmox_backup::tools::runtime::main(async { complete_auth_id_do(param).await })
+}
+
+async fn complete_auth_id_do(param: &HashMap<String, String>) -> Vec<String> {
+
+    let mut result = vec![];
+
+    let repo = match extract_repository_from_map(param) {
+        Some(v) => v,
+        _ => return result,
+    };
+
+    let data = try_get(&repo, "api2/json/access/users?include_tokens=true").await;
+
+    if let Ok(parsed) = serde_json::from_value::<Vec<UserWithTokens>>(data) {
+        for user in parsed {
+            result.push(user.userid.to_string());
+            for token in user.tokens {
+                result.push(token.tokenid.to_string());
+            }
+        }
+    };
+
+    result
+}
+
 use proxmox_backup::client::RemoteChunkReader;
 /// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
 /// async use!
@@ -2013,7 +2044,7 @@ fn main() {
     let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
         .arg_param(&["group", "new-owner"])
         .completion_cb("group", complete_backup_group)
-        .completion_cb("new-owner",  complete_userid)
+        .completion_cb("new-owner",  complete_auth_id)
         .completion_cb("repository", complete_repository);
 
     let cmd_def = CliCommandMap::new()

src/bin/proxmox-backup-manager.rs

@@ -413,29 +413,13 @@ pub fn complete_remote_datastore_name(_arg: &str, param: &HashMap<String, String
 
     let _ = proxmox::try_block!({
         let remote = param.get("remote").ok_or_else(|| format_err!("no remote"))?;
-        let (remote_config, _digest) = config::remote::config()?;
 
-        let remote: config::remote::Remote = remote_config.lookup("remote", &remote)?;
+        let data = crate::tools::runtime::block_on(async move {
+            crate::api2::config::remote::scan_remote_datastores(remote.clone()).await
+        })?;
 
-        let options = HttpClientOptions::new()
+        for item in data {
-            .password(Some(remote.password.clone()))
+            list.push(item.store);
-            .fingerprint(remote.fingerprint.clone());
-
-        let client = HttpClient::new(
-            &remote.host,
-            remote.port.unwrap_or(8007),
-            &remote.userid,
-            options,
-        )?;
-
-        let result = crate::tools::runtime::block_on(client.get("api2/json/admin/datastore", None))?;
-
-        if let Some(data) = result["data"].as_array() {
-            for item in data {
-                if let Some(store) = item["store"].as_str() {
-                    list.push(store.to_owned());
-                }
-            }
         }
 
         Ok(())

src/bin/proxmox-backup-proxy.rs

@@ -1,4 +1,4 @@
-use std::sync::{Arc};
+use std::sync::Arc;
 use std::path::{Path, PathBuf};
 use std::os::unix::io::AsRawFd;
 
@@ -74,6 +74,10 @@ async fn run() -> Result<(), Error> {
         bail!("unable to inititialize syslog - {}", err);
     }
 
+    // Note: To debug early connection error use
+    // PROXMOX_DEBUG=1 ./target/release/proxmox-backup-proxy
+    let debug = std::env::var("PROXMOX_DEBUG").is_ok();
+
     let _ = public_auth_key(); // load with lazy_static
     let _ = csrf_secret(); // load with lazy_static
 
@@ -86,7 +90,6 @@ async fn run() -> Result<(), Error> {
     config.add_alias("xtermjs", "/usr/share/pve-xtermjs");
     config.add_alias("locale", "/usr/share/pbs-i18n");
     config.add_alias("widgettoolkit", "/usr/share/javascript/proxmox-widget-toolkit");
-    config.add_alias("css", "/usr/share/javascript/proxmox-backup/css");
     config.add_alias("docs", "/usr/share/doc/proxmox-backup/html");
 
     let mut indexpath = PathBuf::from(buildcfg::JS_DIR);
@@ -116,25 +119,12 @@ async fn run() -> Result<(), Error> {
     let server = daemon::create_daemon(
         ([0,0,0,0,0,0,0,0], 8007).into(),
         |listener, ready| {
-            let connections = proxmox_backup::tools::async_io::StaticIncoming::from(listener)
-                .map_err(Error::from)
-                .try_filter_map(move |(sock, _addr)| {
-                    let acceptor = Arc::clone(&acceptor);
-                    async move {
-                        sock.set_nodelay(true).unwrap();
 
-                        let _ = set_tcp_keepalive(sock.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME);
+            let connections = accept_connections(listener, acceptor, debug);
-
+            let connections = hyper::server::accept::from_stream(connections);
-                        Ok(tokio_openssl::accept(&acceptor, sock)
-                            .await
-                            .ok() // handshake errors aren't be fatal, so return None to filter
-                        )
-                    }
-                });
-            let connections = proxmox_backup::tools::async_io::HyperAccept(connections);
 
             Ok(ready
-                .and_then(|_| hyper::Server::builder(connections)
+               .and_then(|_| hyper::Server::builder(connections)
                     .serve(rest_server)
                     .with_graceful_shutdown(server::shutdown_future())
                     .map_err(Error::from)
@@ -143,6 +133,7 @@ async fn run() -> Result<(), Error> {
                 .map(|_| ())
             )
         },
+        "proxmox-backup-proxy.service",
     );
 
     server::write_pid(buildcfg::PROXMOX_BACKUP_PROXY_PID_FN)?;
@@ -170,6 +161,72 @@ async fn run() -> Result<(), Error> {
     Ok(())
 }
 
+fn accept_connections(
+    mut listener: tokio::net::TcpListener,
+    acceptor: Arc<openssl::ssl::SslAcceptor>,
+    debug: bool,
+) -> tokio::sync::mpsc::Receiver<Result<tokio_openssl::SslStream<tokio::net::TcpStream>, Error>> {
+
+    const MAX_PENDING_ACCEPTS: usize = 1024;
+
+    let (sender, receiver) = tokio::sync::mpsc::channel(MAX_PENDING_ACCEPTS);
+
+    let accept_counter = Arc::new(());
+
+    tokio::spawn(async move {
+        loop {
+            match listener.accept().await {
+                Err(err) => {
+                    eprintln!("error accepting tcp connection: {}", err);
+                }
+                Ok((sock, _addr)) =>  {
+                    sock.set_nodelay(true).unwrap();
+                    let _ = set_tcp_keepalive(sock.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME);
+                    let acceptor = Arc::clone(&acceptor);
+                    let mut sender = sender.clone();
+
+                    if Arc::strong_count(&accept_counter) > MAX_PENDING_ACCEPTS {
+                        eprintln!("connection rejected - to many open connections");
+                        continue;
+                    }
+
+                    let accept_counter = accept_counter.clone();
+                    tokio::spawn(async move {
+                        let accept_future = tokio::time::timeout(
+                            Duration::new(10, 0), tokio_openssl::accept(&acceptor, sock));
+
+                        let result = accept_future.await;
+
+                        match result {
+                            Ok(Ok(connection)) => {
+                                if let Err(_) = sender.send(Ok(connection)).await {
+                                    if debug {
+                                        eprintln!("detect closed connection channel");
+                                    }
+                                }
+                            }
+                            Ok(Err(err)) => {
+                                if debug {
+                                    eprintln!("https handshake failed - {}", err);
+                                }
+                            }
+                            Err(_) => {
+                                if debug {
+                                    eprintln!("https handshake timeout");
+                                }
+                            }
+                        }
+
+                        drop(accept_counter); // decrease reference count
+                    });
+                }
+            }
+        }
+    });
+
+    receiver
+}
+
 fn start_stat_generator() {
     let abort_future = server::shutdown_future();
     let future = Box::pin(run_stat_generator());
@@ -320,7 +377,7 @@ async fn schedule_datastore_garbage_collection() {
             Err(_) => continue, // could not get lock
         };
 
-        let auth_id = Authid::backup_auth_id();
+        let auth_id = Authid::root_auth_id();
 
         if let Err(err) = crate::server::do_garbage_collection_job(job, datastore, auth_id, Some(event_str), false) {
             eprintln!("unable to start garbage collection job on datastore {} - {}", store, err);
@@ -383,7 +440,7 @@ async fn schedule_datastore_prune() {
                 Err(_) => continue, // could not get lock
             };
 
-            let auth_id = Authid::backup_auth_id().clone();
+            let auth_id = Authid::root_auth_id().clone();
             if let Err(err) = do_prune_job(job, prune_options, store.clone(), &auth_id, Some(event_str)) {
                 eprintln!("unable to start datastore prune job {} - {}", &store, err);
             }
@@ -427,7 +484,7 @@ async fn schedule_datastore_sync_jobs() {
                 Err(_) => continue, // could not get lock
             };
 
-            let auth_id = Authid::backup_auth_id().clone();
+            let auth_id = Authid::root_auth_id().clone();
             if let Err(err) = do_sync_job(job, job_config, &auth_id, Some(event_str)) {
                 eprintln!("unable to start datastore sync job {} - {}", &job_id, err);
             }
@@ -463,7 +520,7 @@ async fn schedule_datastore_verify_jobs() {
         };
 
         let worker_type = "verificationjob";
-        let auth_id = Authid::backup_auth_id().clone();
+        let auth_id = Authid::root_auth_id().clone();
         if check_schedule(worker_type, &event_str, &job_id) {
             let job = match Job::new(&worker_type, &job_id) {
                 Ok(job) => job,
@@ -503,7 +560,7 @@ async fn schedule_task_log_rotate() {
     if let Err(err) = WorkerTask::new_thread(
         worker_type,
         None,
-        Authid::backup_auth_id().clone(),
+        Authid::root_auth_id().clone(),
         false,
         move |worker| {
             job.start(&worker.upid().to_string())?;
@@ -524,16 +581,23 @@ async fn schedule_task_log_rotate() {
                 let mut logrotate = LogRotate::new(buildcfg::API_ACCESS_LOG_FN, true)
                         .ok_or_else(|| format_err!("could not get API access log file names"))?;
 
-                let has_rotated = logrotate.rotate(max_size, None, Some(max_files))?;
+                if logrotate.rotate(max_size, None, Some(max_files))? {
-                if has_rotated {
                     println!("rotated access log, telling daemons to re-open log file");
                     proxmox_backup::tools::runtime::block_on(command_reopen_logfiles())?;
-
                     worker.log(format!("API access log was rotated"));
                 } else {
                     worker.log(format!("API access log was not rotated"));
                 }
 
+                let mut logrotate = LogRotate::new(buildcfg::API_AUTH_LOG_FN, true)
+                        .ok_or_else(|| format_err!("could not get API auth log file names"))?;
+
+                if logrotate.rotate(max_size, None, Some(max_files))? {
+                    worker.log(format!("API authentication log was rotated"));
+                } else {
+                    worker.log(format!("API authentication log was not rotated"));
+                }
+
                 Ok(())
             });
 
@@ -555,16 +619,22 @@ async fn command_reopen_logfiles() -> Result<(), Error> {
     // only care about the most recent daemon instance for each, proxy & api, as other older ones
     // should not respond to new requests anyway, but only finish their current one and then exit.
     let sock = server::our_ctrl_sock();
-    server::send_command(sock, serde_json::json!({
+    let f1 = server::send_command(sock, serde_json::json!({
         "command": "api-access-log-reopen",
-    })).await?;
+    }));
 
     let pid = server::read_pid(buildcfg::PROXMOX_BACKUP_API_PID_FN)?;
     let sock = server::ctrl_sock_from_pid(pid);
-    server::send_command(sock, serde_json::json!({
+    let f2 = server::send_command(sock, serde_json::json!({
         "command": "api-access-log-reopen",
-    })).await?;
+    }));
-    Ok(())
+
+    match futures::join!(f1, f2) {
+        (Err(e1), Err(e2)) => Err(format_err!("reopen commands failed, proxy: {}; api: {}", e1, e2)),
+        (Err(e1), Ok(_)) => Err(format_err!("reopen commands failed, proxy: {}", e1)),
+        (Ok(_), Err(e2)) => Err(format_err!("reopen commands failed, api: {}", e2)),
+        _ => Ok(()),
+    }
 }
 
 async fn run_stat_generator() {

src/bin/proxmox_backup_client/benchmark.rs

@@ -225,7 +225,7 @@ async fn test_upload_speed(
 
     let backup_time = proxmox::tools::time::epoch_i64();
 
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
     record_repository(&repo);
 
     if verbose { eprintln!("Connecting to backup server"); }

src/bin/proxmox_backup_client/catalog.rs

@@ -79,7 +79,7 @@ async fn dump_catalog(param: Value) -> Result<Value, Error> {
         }
     };
 
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
 
     let client = BackupReader::start(
         client,
@@ -153,7 +153,7 @@ async fn dump_catalog(param: Value) -> Result<Value, Error> {
 /// Shell to interactively inspect and restore snapshots.
 async fn catalog_shell(param: Value) -> Result<(), Error> {
     let repo = extract_repository_from_value(&param)?;
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
     let path = tools::required_string_param(&param, "snapshot")?;
     let archive_name = tools::required_string_param(&param, "archive-name")?;
 

src/bin/proxmox_backup_client/mount.rs

@@ -163,7 +163,7 @@ fn mount(
 async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let archive_name = tools::required_string_param(&param, "archive-name")?;
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
 
     let target = param["target"].as_str();
 

src/bin/proxmox_backup_client/task.rs

@@ -48,7 +48,7 @@ async fn task_list(param: Value) -> Result<Value, Error> {
     let output_format = get_output_format(&param);
 
     let repo = extract_repository_from_value(&param)?;
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
 
     let limit = param["limit"].as_u64().unwrap_or(50) as usize;
     let running = !param["all"].as_bool().unwrap_or(false);
@@ -96,7 +96,7 @@ async fn task_log(param: Value) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let upid =  tools::required_string_param(&param, "upid")?;
 
-    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let client = connect(&repo)?;
 
     display_task_log(client, upid, true).await?;
 
@@ -122,7 +122,7 @@ async fn task_stop(param: Value) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let upid_str =  tools::required_string_param(&param, "upid")?;
 
-    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
+    let mut client = connect(&repo)?;
 
     let path = format!("api2/json/nodes/localhost/tasks/{}", upid_str);
     let _ = client.delete(&path, None).await?;

src/buildcfg.rs

@@ -16,9 +16,14 @@ pub const PROXMOX_BACKUP_RUN_DIR: &str = PROXMOX_BACKUP_RUN_DIR_M!();
 /// namespaced directory for persistent logging
 pub const PROXMOX_BACKUP_LOG_DIR: &str = PROXMOX_BACKUP_LOG_DIR_M!();
 
-/// logfile for all API reuests handled by the proxy and privileged API daemons
+/// logfile for all API reuests handled by the proxy and privileged API daemons. Note that not all
+/// failed logins can be logged here with full information, use the auth log for that.
 pub const API_ACCESS_LOG_FN: &str = concat!(PROXMOX_BACKUP_LOG_DIR_M!(), "/api/access.log");
 
+/// logfile for any failed authentication, via ticket or via token, and new successfull ticket
+/// creations. This file can be useful for fail2ban.
+pub const API_AUTH_LOG_FN: &str = concat!(PROXMOX_BACKUP_LOG_DIR_M!(), "/api/auth.log");
+
 /// the PID filename for the unprivileged proxy daemon
 pub const PROXMOX_BACKUP_PROXY_PID_FN: &str = concat!(PROXMOX_BACKUP_RUN_DIR_M!(), "/proxy.pid");
 

src/client/http_client.rs

@@ -405,6 +405,9 @@ impl HttpClient {
     ///
     /// Login is done on demand, so this is only required if you need
     /// access to authentication data in 'AuthInfo'.
+    ///
+    /// Note: tickets a periodially re-newed, so one can use this
+    /// to query changed ticket.
     pub async fn login(&self) -> Result<AuthInfo, Error> {
         if let Some(future) = &self.first_auth {
             future.listen().await?;

src/client/pull.rs

@@ -103,7 +103,7 @@ async fn pull_index_chunks<I: IndexFile>(
 
     let bytes = bytes.load(Ordering::SeqCst);
 
-    worker.log(format!("downloaded {} bytes ({} MiB/s)", bytes, (bytes as f64)/(1024.0*1024.0*elapsed)));
+    worker.log(format!("downloaded {} bytes ({:.2} MiB/s)", bytes, (bytes as f64)/(1024.0*1024.0*elapsed)));
 
     Ok(())
 }
@@ -410,7 +410,8 @@ pub async fn pull_group(
 
     list.sort_unstable_by(|a, b| a.backup_time.cmp(&b.backup_time));
 
-    let auth_info = client.login().await?;
+    client.login().await?; // make sure auth is complete
+
     let fingerprint = client.fingerprint();
 
     let last_sync = tgt_store.last_successful_backup(group)?;
@@ -447,6 +448,9 @@ pub async fn pull_group(
             if last_sync_time > backup_time { continue; }
         }
 
+        // get updated auth_info (new tickets)
+        let auth_info = client.login().await?;
+
         let options = HttpClientOptions::new()
             .password(Some(auth_info.ticket.clone()))
             .fingerprint(fingerprint.clone());
@@ -524,7 +528,16 @@ pub async fn pull_store(
     for (groups_done, item) in list.into_iter().enumerate() {
         let group = BackupGroup::new(&item.backup_type, &item.backup_id);
 
-        let (owner, _lock_guard) = tgt_store.create_locked_backup_group(&group, &auth_id)?;
+        let (owner, _lock_guard) = match tgt_store.create_locked_backup_group(&group, &auth_id) {
+            Ok(result) => result,
+            Err(err) => {
+                worker.log(format!("sync group {}/{} failed - group lock failed: {}",
+                                   item.backup_type, item.backup_id, err));
+                errors = true; // do not stop here, instead continue
+                continue;
+            }
+        };
+
         // permission check
         if auth_id != owner { // only the owner is allowed to create additional snapshots
             worker.log(format!("sync group {}/{} failed - owner check failed ({} != {})",

src/config/datastore.rs

@@ -32,6 +32,14 @@ pub const DIR_NAME_SCHEMA: Schema = StringSchema::new("Directory name").schema()
         path: {
             schema: DIR_NAME_SCHEMA,
         },
+        "notify-user": {
+            optional: true,
+            type: Userid,
+        },
+        "notify": {
+            optional: true,
+            schema: DATASTORE_NOTIFY_STRING_SCHEMA,
+        },
         comment: {
             optional: true,
             schema: SINGLE_LINE_COMMENT_SCHEMA,
@@ -101,6 +109,12 @@ pub struct DataStoreConfig {
     /// If enabled, all backups will be verified right after completion.
     #[serde(skip_serializing_if="Option::is_none")]
     pub verify_new: Option<bool>,
+    /// Send job email notification to this user
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub notify_user: Option<Userid>,
+    /// Send notification only for job errors
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub notify: Option<String>,
 }
 
 fn init() -> SectionConfig {

src/config/remote.rs

@@ -44,7 +44,7 @@ pub const REMOTE_PASSWORD_SCHEMA: Schema = StringSchema::new("Password or auth t
             description: "The (optional) port",
             type: u16,
         },
-        userid: {
+        "auth-id": {
             type: Authid,
         },
         password: {
@@ -57,6 +57,7 @@ pub const REMOTE_PASSWORD_SCHEMA: Schema = StringSchema::new("Password or auth t
     }
 )]
 #[derive(Serialize,Deserialize)]
+#[serde(rename_all = "kebab-case")]
 /// Remote properties.
 pub struct Remote {
     pub name: String,
@@ -65,7 +66,7 @@ pub struct Remote {
     pub host: String,
     #[serde(skip_serializing_if="Option::is_none")]
     pub port: Option<u16>,
-    pub userid: Authid,
+    pub auth_id: Authid,
     #[serde(skip_serializing_if="String::is_empty")]
     #[serde(with = "proxmox::tools::serde::string_as_base64")]
     pub password: String,

src/pxar/create.rs

@@ -89,7 +89,21 @@ struct HardLinkInfo {
     st_ino: u64,
 }
 
-/// In case we want to collect them or redirect them we can just add this here:
+/// TODO: make a builder for the create_archive call for fewer parameters and add a method to add a
+/// logger which does not write to stderr.
+struct Logger;
+
+impl std::io::Write for Logger {
+    fn write(&mut self, data: &[u8]) -> io::Result<usize> {
+        std::io::stderr().write(data)
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        std::io::stderr().flush()
+    }
+}
+
+/// And the error case.
 struct ErrorReporter;
 
 impl std::io::Write for ErrorReporter {
@@ -116,6 +130,7 @@ struct Archiver<'a, 'b> {
     device_set: Option<HashSet<u64>>,
     hardlinks: HashMap<HardLinkInfo, (PathBuf, LinkOffset)>,
     errors: ErrorReporter,
+    logger: Logger,
     file_copy_buffer: Vec<u8>,
 }
 
@@ -181,6 +196,7 @@ where
         device_set,
         hardlinks: HashMap::new(),
         errors: ErrorReporter,
+        logger: Logger,
         file_copy_buffer: vec::undefined(4 * 1024 * 1024),
     };
 
@@ -632,6 +648,7 @@ impl<'a, 'b> Archiver<'a, 'b> {
         }
 
         let result = if skip_contents {
+            writeln!(self.logger, "skipping mount point: {:?}", self.path)?;
             Ok(())
         } else {
             self.archive_dir_contents(&mut encoder, dir, false)
@@ -657,7 +674,12 @@ impl<'a, 'b> Archiver<'a, 'b> {
         let mut remaining = file_size;
         let mut out = encoder.create_file(metadata, file_name, file_size)?;
         while remaining != 0 {
-            let mut got = file.read(&mut self.file_copy_buffer[..])?;
+            let mut got = match file.read(&mut self.file_copy_buffer[..]) {
+                Ok(0) => break,
+                Ok(got) => got,
+                Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue,
+                Err(err) => bail!(err),
+            };
             if got as u64 > remaining {
                 self.report_file_grew_while_reading()?;
                 got = remaining as usize;

src/server/email_notifications.rs

@@ -4,14 +4,18 @@ use serde_json::json;
 use handlebars::{Handlebars, Helper, Context, RenderError, RenderContext, Output, HelperResult};
 
 use proxmox::tools::email::sendmail;
+use proxmox::api::schema::parse_property_string;
 
 use crate::{
+    config::datastore::DataStoreConfig,
     config::verify::VerificationJobConfig,
     config::sync::SyncJobConfig,
     api2::types::{
         APTUpdateInfo,
         GarbageCollectionStatus,
         Userid,
+        Notify,
+        DatastoreNotify,
     },
     tools::format::HumanByte,
 };
@@ -188,11 +192,21 @@ fn send_job_status_mail(
 
 pub fn send_gc_status(
     email: &str,
+    notify: DatastoreNotify,
     datastore: &str,
     status: &GarbageCollectionStatus,
     result: &Result<(), Error>,
 ) -> Result<(), Error> {
 
+    match notify.gc {
+        None => { /* send notifications by default */ },
+        Some(notify) => {
+            if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
+                return Ok(());
+            }
+        }
+    }
+
     let (fqdn, port) = get_server_url();
     let mut data = json!({
         "datastore": datastore,
@@ -237,6 +251,7 @@ pub fn send_gc_status(
 
 pub fn send_verify_status(
     email: &str,
+    notify: DatastoreNotify,
     job: VerificationJobConfig,
     result: &Result<Vec<String>, Error>,
 ) -> Result<(), Error> {
@@ -248,8 +263,11 @@ pub fn send_verify_status(
         "port": port,
     });
 
+    let mut result_is_ok = false;
+
     let text = match result {
         Ok(errors) if errors.is_empty() => {
+            result_is_ok = true;
             HANDLEBARS.render("verify_ok_template", &data)?
         }
         Ok(errors) => {
@@ -262,6 +280,15 @@ pub fn send_verify_status(
         }
     };
 
+    match notify.verify {
+        None => { /* send notifications by default */ },
+        Some(notify) => {
+            if notify == Notify::Never || (result_is_ok && notify == Notify::Error) {
+                return Ok(());
+            }
+        }
+    }
+
     let subject = match result {
         Ok(errors) if errors.is_empty() => format!(
             "Verify Datastore '{}' successful",
@@ -280,10 +307,20 @@ pub fn send_verify_status(
 
 pub fn send_sync_status(
     email: &str,
+    notify: DatastoreNotify,
     job: &SyncJobConfig,
     result: &Result<(), Error>,
 ) -> Result<(), Error> {
 
+    match notify.sync {
+        None => { /* send notifications by default */ },
+        Some(notify) => {
+            if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
+                return Ok(());
+            }
+        }
+    }
+
     let (fqdn, port) = get_server_url();
     let mut data = json!({
         "job": job,
@@ -360,16 +397,10 @@ pub fn send_updates_available(
 }
 
 /// Lookup users email address
-///
+fn lookup_user_email(userid: &Userid) -> Option<String> {
-/// For "backup@pam", this returns the address from "root@pam".
-pub fn lookup_user_email(userid: &Userid) -> Option<String> {
 
     use crate::config::user::{self, User};
 
-    if userid == Userid::backup_userid() {
-        return lookup_user_email(Userid::root_userid());
-    }
-
     if let Ok(user_config) = user::cached_config() {
         if let Ok(user) = user_config.lookup::<User>("user", userid.as_str()) {
             return user.email.clone();
@@ -379,6 +410,41 @@ pub fn lookup_user_email(userid: &Userid) -> Option<String> {
     None
 }
 
+/// Lookup Datastore notify settings
+pub fn lookup_datastore_notify_settings(
+    store: &str,
+) -> (Option<String>, DatastoreNotify) {
+
+    let mut email = None;
+
+    let notify = DatastoreNotify { gc: None, verify: None, sync: None };
+
+    let (config, _digest) = match crate::config::datastore::config() {
+        Ok(result) => result,
+        Err(_) => return (email, notify),
+    };
+
+    let config: DataStoreConfig = match config.lookup("datastore", store) {
+        Ok(result) => result,
+        Err(_) => return (email, notify),
+    };
+
+    email = match config.notify_user {
+        Some(ref userid) => lookup_user_email(userid),
+        None => lookup_user_email(Userid::root_userid()),
+    };
+
+    let notify_str = config.notify.unwrap_or(String::new());
+
+    if let Ok(value) = parse_property_string(&notify_str, &DatastoreNotify::API_SCHEMA) {
+        if let Ok(notify) = serde_json::from_value(value) {
+            return (email, notify);
+        }
+    }
+
+    (email, notify)
+}
+
 // Handlerbar helper functions
 
 fn handlebars_humam_bytes_helper(

src/server/gc_job.rs

@@ -17,10 +17,10 @@ pub fn do_garbage_collection_job(
     to_stdout: bool,
 ) -> Result<String, Error> {
 
-    let email = crate::server::lookup_user_email(auth_id.user());
-
     let store = datastore.name().to_string();
 
+    let (email, notify) = crate::server::lookup_datastore_notify_settings(&store);
+
     let worker_type = job.jobtype().to_string();
     let upid_str = WorkerTask::new_thread(
         &worker_type,
@@ -50,7 +50,7 @@ pub fn do_garbage_collection_job(
 
             if let Some(email) = email {
                 let gc_status = datastore.last_gc_status();
-                if let Err(err) = crate::server::send_gc_status(&email, &store, &gc_status, &result) {
+                if let Err(err) = crate::server::send_gc_status(&email, notify, &store, &gc_status, &result) {
                     eprintln!("send gc notification failed: {}", err);
                 }
             }

src/server/rest.rs

@@ -164,6 +164,15 @@ fn log_response(
             ));
     }
 }
+pub fn auth_logger() -> Result<FileLogger, Error> {
+    let logger_options = tools::FileLogOptions {
+        append: true,
+        prefix_time: true,
+        owned_by_backup: true,
+        ..Default::default()
+    };
+    FileLogger::new(crate::buildcfg::API_AUTH_LOG_FN, logger_options)
+}
 
 fn get_proxied_peer(headers: &HeaderMap) -> Option<std::net::SocketAddr> {
     lazy_static! {
@@ -687,6 +696,10 @@ async fn handle_request(
                 match auth_result {
                     Ok(authid) => rpcenv.set_auth_id(Some(authid.to_string())),
                     Err(err) => {
+                        let peer = peer.ip();
+                        auth_logger()?
+                            .log(format!("authentication failure; rhost={} msg={}", peer, err));
+
                         // always delay unauthorized calls by 3 seconds (from start of request)
                         let err = http_err!(UNAUTHORIZED, "authentication failed - {}", err);
                         tokio::time::delay_until(Instant::from_std(delay_unauth_time)).await;

src/server/verify_job.rs

@@ -48,13 +48,15 @@ pub fn do_verification_job(
         }
     };
 
-    let email = crate::server::lookup_user_email(auth_id.user());
+    let (email, notify) = crate::server::lookup_datastore_notify_settings(&verification_job.store);
 
-    let job_id = job.jobname().to_string();
+    let job_id = format!("{}:{}",
+                         &verification_job.store,
+                         job.jobname());
     let worker_type = job.jobtype().to_string();
     let upid_str = WorkerTask::new_thread(
         &worker_type,
-        Some(job.jobname().to_string()),
+        Some(job_id.clone()),
         auth_id.clone(),
         false,
         move |worker| {
@@ -84,7 +86,7 @@ pub fn do_verification_job(
             }
 
             if let Some(email) = email {
-                if let Err(err) = crate::server::send_verify_status(&email, verification_job, &result) {
+                if let Err(err) = crate::server::send_verify_status(&email, notify, verification_job, &result) {
                     eprintln!("send verify notification failed: {}", err);
                 }
             }

src/server/worker_task.rs

@@ -55,7 +55,9 @@ pub async fn worker_is_active(upid: &UPID) -> Result<bool, Error> {
     let sock = server::ctrl_sock_from_pid(upid.pid);
     let cmd = json!({
         "command": "worker-task-status",
-        "upid": upid.to_string(),
+        "args": {
+            "upid": upid.to_string(),
+        },
     });
     let status = super::send_command(sock, cmd).await?;
 
@@ -127,7 +129,9 @@ pub async fn abort_worker(upid: UPID) -> Result<(), Error> {
     let sock = server::ctrl_sock_from_pid(upid.pid);
     let cmd = json!({
         "command": "worker-task-abort",
-        "upid": upid.to_string(),
+        "args": {
+            "upid": upid.to_string(),
+        },
     });
     super::send_command(sock, cmd).map_ok(|_| ()).await
 }

src/tools/apt.rs

@@ -128,19 +128,26 @@ fn get_changelog_url(
             None => bail!("incompatible filename, doesn't match regex")
         };
 
-        return Ok(format!("http://download.proxmox.com/{}/{}_{}.changelog",
+        if component == "pbs-enterprise" {
-                          base, package, version));
+            return Ok(format!("https://enterprise.proxmox.com/{}/{}_{}.changelog",
+                              base, package, version));
+        } else {
+            return Ok(format!("http://download.proxmox.com/{}/{}_{}.changelog",
+                              base, package, version));
+        }
     }
 
     bail!("unknown origin ({}) or component ({})", origin, component)
 }
 
 pub struct FilterData<'a> {
-    // this is version info returned by APT
+    /// package name
+    pub package: &'a str,
+    /// this is version info returned by APT
     pub installed_version: Option<&'a str>,
     pub candidate_version: &'a str,
 
-    // this is the version info the filter is supposed to check
+    /// this is the version info the filter is supposed to check
     pub active_version: &'a str,
 }
 
@@ -270,6 +277,7 @@ where
         let mut long_desc = "".to_owned();
 
         let fd = FilterData {
+            package: package.as_str(),
             installed_version: current_version.as_deref(),
             candidate_version: &candidate_version,
             active_version: &version,

src/tools/async_io.rs

@@ -16,18 +16,18 @@ pub enum EitherStream<L, R> {
     Right(R),
 }
 
-impl<L: AsyncRead, R: AsyncRead> AsyncRead for EitherStream<L, R> {
+impl<L: AsyncRead + Unpin, R: AsyncRead + Unpin> AsyncRead for EitherStream<L, R> {
     fn poll_read(
         self: Pin<&mut Self>,
         cx: &mut Context,
         buf: &mut [u8],
     ) -> Poll<Result<usize, io::Error>> {
-        match unsafe { self.get_unchecked_mut() } {
+        match self.get_mut() {
             EitherStream::Left(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_read(cx, buf)
+                Pin::new(s).poll_read(cx, buf)
             }
             EitherStream::Right(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_read(cx, buf)
+                Pin::new(s).poll_read(cx, buf)
             }
         }
     }
@@ -47,51 +47,51 @@ impl<L: AsyncRead, R: AsyncRead> AsyncRead for EitherStream<L, R> {
     where
         B: bytes::BufMut,
     {
-        match unsafe { self.get_unchecked_mut() } {
+        match self.get_mut() {
             EitherStream::Left(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_read_buf(cx, buf)
+                Pin::new(s).poll_read_buf(cx, buf)
             }
             EitherStream::Right(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_read_buf(cx, buf)
+                Pin::new(s).poll_read_buf(cx, buf)
             }
         }
     }
 }
 
-impl<L: AsyncWrite, R: AsyncWrite> AsyncWrite for EitherStream<L, R> {
+impl<L: AsyncWrite + Unpin, R: AsyncWrite + Unpin> AsyncWrite for EitherStream<L, R> {
     fn poll_write(
         self: Pin<&mut Self>,
         cx: &mut Context,
         buf: &[u8],
     ) -> Poll<Result<usize, io::Error>> {
-        match unsafe { self.get_unchecked_mut() } {
+        match self.get_mut() {
             EitherStream::Left(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_write(cx, buf)
+                Pin::new(s).poll_write(cx, buf)
             }
             EitherStream::Right(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_write(cx, buf)
+                Pin::new(s).poll_write(cx, buf)
             }
         }
     }
 
     fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Result<(), io::Error>> {
-        match unsafe { self.get_unchecked_mut() } {
+        match self.get_mut() {
             EitherStream::Left(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_flush(cx)
+                Pin::new(s).poll_flush(cx)
             }
             EitherStream::Right(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_flush(cx)
+                Pin::new(s).poll_flush(cx)
             }
         }
     }
 
     fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Result<(), io::Error>> {
-        match unsafe { self.get_unchecked_mut() } {
+        match self.get_mut() {
             EitherStream::Left(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_shutdown(cx)
+                Pin::new(s).poll_shutdown(cx)
             }
             EitherStream::Right(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_shutdown(cx)
+                Pin::new(s).poll_shutdown(cx)
             }
         }
     }
@@ -104,12 +104,12 @@ impl<L: AsyncWrite, R: AsyncWrite> AsyncWrite for EitherStream<L, R> {
     where
         B: bytes::Buf,
     {
-        match unsafe { self.get_unchecked_mut() } {
+        match self.get_mut() {
             EitherStream::Left(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_write_buf(cx, buf)
+                Pin::new(s).poll_write_buf(cx, buf)
             }
             EitherStream::Right(ref mut s) => {
-                unsafe { Pin::new_unchecked(s) }.poll_write_buf(cx, buf)
+                Pin::new(s).poll_write_buf(cx, buf)
             }
         }
     }
@@ -180,7 +180,7 @@ pub struct HyperAccept<T>(pub T);
 
 impl<T, I> hyper::server::accept::Accept for HyperAccept<T>
 where
-    T: TryStream<Ok = I>,
+    T: TryStream<Ok = I> + Unpin,
     I: AsyncRead + AsyncWrite,
 {
     type Conn = I;
@@ -190,7 +190,7 @@ where
         self: Pin<&mut Self>,
         cx: &mut Context,
     ) -> Poll<Option<Result<Self::Conn, Self::Error>>> {
-        let this = unsafe { self.map_unchecked_mut(|this| &mut this.0) };
+        let this = Pin::new(&mut self.get_mut().0);
         this.try_poll_next(cx)
     }
 }

src/tools/daemon.rs

@@ -12,6 +12,7 @@ use std::task::{Context, Poll};
 use std::path::PathBuf;
 
 use anyhow::{bail, format_err, Error};
+use futures::future::{self, Either};
 
 use proxmox::tools::io::{ReadExt, WriteExt};
 
@@ -259,10 +260,11 @@ impl Future for NotifyReady {
 pub async fn create_daemon<F, S>(
     address: std::net::SocketAddr,
     create_service: F,
+    service_name: &str,
 ) -> Result<(), Error>
 where
     F: FnOnce(tokio::net::TcpListener, NotifyReady) -> Result<S, Error>,
-    S: Future<Output = ()>,
+    S: Future<Output = ()> + Unpin,
 {
     let mut reloader = Reloader::new()?;
 
@@ -271,11 +273,19 @@ where
         move || async move { Ok(tokio::net::TcpListener::bind(&address).await?) },
     ).await?;
 
-    create_service(listener, NotifyReady)?.await;
+    let server_future = create_service(listener, NotifyReady)?;
+    let shutdown_future = server::shutdown_future();
+
+    let finish_future = match future::select(server_future, shutdown_future).await {
+        Either::Left((_, _)) => {
+            crate::tools::request_shutdown(); // make sure we are in shutdown mode
+            None
+        }
+        Either::Right((_, server_future)) => Some(server_future),
+    };
 
     let mut reloader = Some(reloader);
 
-    crate::tools::request_shutdown(); // make sure we are in shutdown mode
     if server::is_reload_request() {
         log::info!("daemon reload...");
         if let Err(e) = systemd_notify(SystemdNotify::Reloading) {
@@ -288,9 +298,43 @@ where
     } else {
         log::info!("daemon shutting down...");
     }
+
+    if let Some(future) = finish_future {
+        future.await;
+    }
+
+    // FIXME: this is a hack, replace with sd_notify_barrier when available
+    if server::is_reload_request() {
+        wait_service_is_active(service_name).await?;
+    }
+
+    log::info!("daemon shut down...");
     Ok(())
 }
 
+// hack, do not use if unsure!
+async fn wait_service_is_active(service: &str) -> Result<(), Error> {
+    tokio::time::delay_for(std::time::Duration::new(1, 0)).await;
+    loop {
+        let text = match tokio::process::Command::new("systemctl")
+            .args(&["is-active", service])
+            .output()
+            .await
+        {
+            Ok(output) => match String::from_utf8(output.stdout) {
+                Ok(text) => text,
+                Err(err) => bail!("output of 'systemctl is-active' not valid UTF-8 - {}", err),
+            },
+            Err(err) => bail!("executing 'systemctl is-active' failed - {}", err),
+        };
+
+        if text.trim().trim_start() != "reloading" {
+            return Ok(());
+        }
+        tokio::time::delay_for(std::time::Duration::new(5, 0)).await;
+    }
+}
+
 #[link(name = "systemd")]
 extern "C" {
     fn sd_notify(unset_environment: c_int, state: *const c_char) -> c_int;

src/tools/http.rs

@@ -2,6 +2,7 @@ use anyhow::{Error, format_err, bail};
 use lazy_static::lazy_static;
 use std::task::{Context, Poll};
 use std::os::unix::io::AsRawFd;
+use std::collections::HashMap;
 
 use hyper::{Uri, Body};
 use hyper::client::{Client, HttpConnector};
@@ -26,8 +27,21 @@ lazy_static! {
     };
 }
 
-pub async fn get_string(uri: &str) -> Result<String, Error> {
+pub async fn get_string(uri: &str, extra_headers: Option<&HashMap<String, String>>) -> Result<String, Error> {
-    let res = HTTP_CLIENT.get(uri.parse()?).await?;
+    let mut request = Request::builder()
+        .method("GET")
+        .uri(uri)
+        .header("User-Agent", "proxmox-backup-client/1.0");
+
+    if let Some(hs) = extra_headers {
+        for (h, v) in hs.iter() {
+            request = request.header(h, v);
+        }
+    }
+
+    let request = request.body(Body::empty())?;
+
+    let res = HTTP_CLIENT.request(request).await?;
 
     let status = res.status();
     if !status.is_success() {
@@ -115,7 +129,12 @@ impl hyper::service::Service<Uri> for HttpsConnector {
                 .to_string();
 
             let config = this.ssl_connector.configure();
-            let conn = this.http.call(dst).await?;
+            let dst_str = dst.to_string(); // for error messages
+            let conn = this
+                .http
+                .call(dst)
+                .await
+                .map_err(|err| format_err!("error connecting to {} - {}", dst_str, err))?;
 
             let _ = set_tcp_keepalive(conn.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME);
 

src/tools/logrotate.rs

@@ -92,15 +92,15 @@ impl LogRotate {
         if filenames.is_empty() {
             return Ok(()); // no file means nothing to rotate
         }
+        let count = filenames.len() + 1;
 
         let mut next_filename = self.base_path.clone().canonicalize()?.into_os_string();
         next_filename.push(format!(".{}", filenames.len()));
-        if self.compress {
+        if self.compress && count > 2 {
             next_filename.push(".zst");
         }
 
         filenames.push(PathBuf::from(next_filename));
-        let count = filenames.len();
 
         for i in (0..count-1).rev() {
             if self.compress

src/tools/runtime.rs

@@ -7,6 +7,7 @@ use std::task::{Context, Poll, RawWaker, Waker};
 use std::thread::{self, Thread};
 
 use lazy_static::lazy_static;
+use pin_utils::pin_mut;
 use tokio::runtime::{self, Runtime};
 
 thread_local! {
@@ -154,9 +155,8 @@ pub fn main<F: Future>(fut: F) -> F::Output {
     block_on(fut)
 }
 
-fn block_on_local_future<F: Future>(mut fut: F) -> F::Output {
+fn block_on_local_future<F: Future>(fut: F) -> F::Output {
-    use std::pin::Pin;
+    pin_mut!(fut);
-    let mut fut = unsafe { Pin::new_unchecked(&mut fut) };
 
     let waker = Arc::new(thread::current());
     let waker = thread_waker_clone(Arc::into_raw(waker) as *const ());

www/Application.js

@@ -36,8 +36,7 @@ Ext.define('PBS.Application', {
 	    xtype: view,
 	});
 	if (skipCheck !== true) {
-	    // fixme:
+	    Proxmox.Utils.checked_command(Ext.emptyFn);
-	    // Proxmox.Utils.checked_command(function() {}); // display subscription status
 	}
     },
 

www/Dashboard.js

@@ -332,12 +332,13 @@ Ext.define('PBS.Dashboard', {
 		Ext.String.format(gettext('{0} days'), '{days}') + ')',
 	    },
 	    xtype: 'pbsTaskSummary',
+	    height: 200,
 	    reference: 'tasksummary',
 	},
 	{
 	    iconCls: 'fa fa-ticket',
 	    title: 'Subscription',
-	    height: 166,
+	    height: 200,
 	    reference: 'subscription',
 	    xtype: 'pbsSubscriptionInfo',
 	},
@@ -394,7 +395,7 @@ Ext.define('PBS.dashboard.SubscriptionInfo', {
 		break;
 	    case 0:
 		icon = 'times-circle critical';
-		message = gettext('This node does not have a subscription.');
+		message = gettext('<h1>No valid subscription</h1>' + PBS.Utils.noSubKeyHtml);
 		break;
 	    default:
 		throw 'invalid subscription status';

www/MainView.js

@@ -202,11 +202,6 @@ Ext.define('PBS.MainView', {
 		    padding: '0 0 0 5',
 		    xtype: 'versioninfo',
 		},
-		{
-		    padding: 5,
-		    html: '<a href="https://bugzilla.proxmox.com" target="_blank">BETA</a>',
-		    baseCls: 'x-plain',
-		},
 		{
 		    flex: 1,
 		    baseCls: 'x-plain',

www/Makefile

@@ -5,8 +5,10 @@ IMAGES := \
 	images/proxmox_logo.png
 
 JSSRC=							\
+	Utils.js					\
 	form/UserSelector.js				\
 	form/TokenSelector.js				\
+	form/AuthidSelector.js				\
 	form/RemoteSelector.js				\
 	form/DataStoreSelector.js			\
 	form/CalendarEvent.js				\
@@ -21,11 +23,13 @@ JSSRC=							\
 	config/VerifyView.js				\
 	window/ACLEdit.js				\
 	window/BackupFileDownloader.js			\
+	window/BackupGroupChangeOwner.js		\
 	window/CreateDirectory.js			\
 	window/DataStoreEdit.js				\
 	window/FileBrowser.js				\
 	window/NotesEdit.js				\
 	window/RemoteEdit.js				\
+	window/NotifyOptions.js				\
 	window/SyncJobEdit.js				\
 	window/UserEdit.js				\
 	window/UserPassword.js				\
@@ -37,20 +41,25 @@ JSSRC=							\
 	dashboard/RunningTasks.js			\
 	dashboard/TaskSummary.js			\
 	panel/Tasks.js					\
-	Utils.js					\
+	panel/XtermJsConsole.js				\
-	AccessControlPanel.js				\
+	panel/AccessControl.js				\
+	panel/StorageAndDisks.js			\
+	panel/UsageChart.js				\
 	ZFSList.js					\
 	DirectoryList.js				\
 	LoginView.js					\
 	VersionInfo.js					\
 	SystemConfiguration.js				\
 	Subscription.js					\
-	DataStoreSummary.js				\
+	datastore/Summary.js				\
-	DataStoreNotes.js				\
+	datastore/Notes.js				\
-	DataStorePruneAndGC.js				\
+	datastore/PruneAndGC.js				\
-	DataStorePrune.js				\
+	datastore/Prune.js				\
-	DataStoreContent.js				\
+	datastore/Content.js				\
-	DataStorePanel.js				\
+	datastore/OptionView.js				\
+	datastore/Panel.js				\
+	datastore/DataStoreListSummary.js		\
+	datastore/DataStoreList.js			\
 	ServerStatus.js					\
 	ServerAdministration.js				\
 	Dashboard.js					\

www/NavigationTree.js

@@ -56,24 +56,16 @@ Ext.define('PBS.store.NavigationStore', {
 		leaf: false,
 		children: [
 		    {
-			text: gettext('Disks'),
+			text: gettext('Shell'),
+			iconCls: 'fa fa-terminal',
+			path: 'pbsXtermJsConsole',
+			leaf: true,
+		    },
+		    {
+			text: gettext('Storage / Disks'),
 			iconCls: 'fa fa-hdd-o',
-			path: 'pmxDiskList',
+			path: 'pbsStorageAndDiskPanel',
-			leaf: false,
+			leaf: true,
-			children: [
-			    {
-				text: Proxmox.Utils.directoryText,
-				iconCls: 'fa fa-folder',
-				path: 'pbsDirectoryList',
-				leaf: true,
-			    },
-			    {
-				text: "ZFS",
-				iconCls: 'fa fa-th-large',
-				path: 'pbsZFSList',
-				leaf: true,
-			    },
-			],
 		    },
 		],
 	    },
@@ -81,6 +73,7 @@ Ext.define('PBS.store.NavigationStore', {
 		text: gettext('Datastore'),
 		iconCls: 'fa fa-archive',
 		id: 'datastores',
+		path: 'pbsDataStores',
 		expanded: true,
 		expandable: false,
 		leaf: false,
@@ -168,9 +161,6 @@ Ext.define('PBS.view.main.NavigationTree', {
 
     listeners: {
 	itemclick: function(tl, info) {
-	    if (info.node.data.id === 'datastores') {
-		return false;
-	    }
 	    if (info.node.data.id === 'addbutton') {
 		let me = this;
 		Ext.create('PBS.DataStoreEdit', {

www/OnlineHelpInfo.js

@@ -3,30 +3,138 @@ const proxmoxOnlineHelpInfo = {
     "link": "/docs/index.html",
     "title": "Proxmox Backup Server Documentation Index"
   },
+  "id1": {
+    "link": "/docs/calendarevents.html#id1",
+    "title": "Calendar Events"
+  },
+  "id2": {
+    "link": "/docs/backup-client.html#id2",
+    "title": "Encryption"
+  },
+  "changing-backup-owner": {
+    "link": "/docs/backup-client.html#changing-backup-owner",
+    "title": "Changing the Owner of a Backup Group"
+  },
   "backup-pruning": {
     "link": "/docs/backup-client.html#backup-pruning",
     "title": "Pruning and Removing Backups"
   },
+  "id3": {
+    "link": "/docs/backup-client.html#id3",
+    "title": "Garbage Collection"
+  },
+  "pxar-format": {
+    "link": "/docs/file-formats.html#pxar-format",
+    "title": "Proxmox File Archive Format (``.pxar``)"
+  },
+  "sysadmin-package-repositories": {
+    "link": "/docs/package-repositories.html#sysadmin-package-repositories",
+    "title": "Debian Package Repositories"
+  },
+  "get-help": {
+    "link": "/docs/introduction.html#get-help",
+    "title": "Getting Help"
+  },
   "chapter-zfs": {
     "link": "/docs/sysadmin.html#chapter-zfs",
     "title": "ZFS on Linux"
   },
+  "local-zfs-special-device": {
+    "link": "/docs/sysadmin.html#local-zfs-special-device",
+    "title": "ZFS Special Device"
+  },
+  "maintenance-pruning": {
+    "link": "/docs/maintenance.html#maintenance-pruning",
+    "title": "Pruning"
+  },
+  "maintenance-gc": {
+    "link": "/docs/maintenance.html#maintenance-gc",
+    "title": "Garbage Collection"
+  },
+  "maintenance-verification": {
+    "link": "/docs/maintenance.html#maintenance-verification",
+    "title": "Verification"
+  },
+  "maintenance-notification": {
+    "link": "/docs/maintenance.html#maintenance-notification",
+    "title": "Notifications"
+  },
   "backup-remote": {
     "link": "/docs/managing-remotes.html#backup-remote",
-    "title": ":term:`Remote`"
+    "title": "Remote"
   },
   "syncjobs": {
     "link": "/docs/managing-remotes.html#syncjobs",
     "title": "Sync Jobs"
   },
+  "sysadmin-network-configuration": {
+    "link": "/docs/network-management.html#sysadmin-network-configuration",
+    "title": "Network Management"
+  },
+  "pve-integration": {
+    "link": "/docs/pve-integration.html#pve-integration",
+    "title": "`Proxmox VE`_ Integration"
+  },
+  "rst-primer": {
+    "link": "/docs/reStructuredText-primer.html#rst-primer",
+    "title": "reStructuredText Primer"
+  },
+  "rst-inline-markup": {
+    "link": "/docs/reStructuredText-primer.html#rst-inline-markup",
+    "title": "Inline markup"
+  },
+  "rst-literal-blocks": {
+    "link": "/docs/reStructuredText-primer.html#rst-literal-blocks",
+    "title": "Literal blocks"
+  },
+  "rst-doctest-blocks": {
+    "link": "/docs/reStructuredText-primer.html#rst-doctest-blocks",
+    "title": "Doctest blocks"
+  },
+  "rst-tables": {
+    "link": "/docs/reStructuredText-primer.html#rst-tables",
+    "title": "Tables"
+  },
+  "rst-field-lists": {
+    "link": "/docs/reStructuredText-primer.html#rst-field-lists",
+    "title": "Field Lists"
+  },
+  "rst-roles-alt": {
+    "link": "/docs/reStructuredText-primer.html#rst-roles-alt",
+    "title": "Roles"
+  },
+  "rst-directives": {
+    "link": "/docs/reStructuredText-primer.html#rst-directives",
+    "title": "Directives"
+  },
+  "html-meta": {
+    "link": "/docs/reStructuredText-primer.html#html-meta",
+    "title": "HTML Metadata"
+  },
+  "storage-disk-management": {
+    "link": "/docs/storage.html#storage-disk-management",
+    "title": "Disk Management"
+  },
   "datastore-intro": {
     "link": "/docs/storage.html#datastore-intro",
-    "title": ":term:`DataStore`"
+    "title": "Datastore"
+  },
+  "storage-datastore-create": {
+    "link": "/docs/storage.html#storage-datastore-create",
+    "title": "Creating a Datastore"
+  },
+  "sysadmin-host-administration": {
+    "link": "/docs/sysadmin.html#sysadmin-host-administration",
+    "title": "Host System Administration"
   },
   "user-mgmt": {
     "link": "/docs/user-management.html#user-mgmt",
     "title": "User Management"
   },
+  "user-tokens": {
+    "link": "/docs/user-management.html#user-tokens",
+    "title": "API Tokens"
+  },
   "user-acl": {
     "link": "/docs/user-management.html#user-acl",
     "title": "Access Control"

www/ServerAdministration.js

@@ -7,6 +7,8 @@ Ext.define('PBS.ServerAdministration', {
     border: true,
     defaults: { border: false },
 
+    tools: [PBS.Utils.get_help_tool("sysadmin-host-administration")],
+
     controller: {
 	xclass: 'Ext.app.ViewController',
 

www/ServerStatus.js

@@ -51,6 +51,78 @@ Ext.define('PBS.ServerStatus', {
 
     scrollable: true,
 
+    showVersions: function() {
+	let me = this;
+
+	// Note: use simply text/html here, as ExtJS grid has problems with cut&paste
+	let panel = Ext.createWidget('component', {
+	    autoScroll: true,
+	    id: 'pkgversions',
+	    padding: 5,
+	    style: {
+		'background-color': 'white',
+		'white-space': 'pre',
+		'font-family': 'monospace',
+	    },
+	});
+
+	let win = Ext.create('Ext.window.Window', {
+	    title: gettext('Package versions'),
+	    width: 600,
+	    height: 600,
+	    layout: 'fit',
+	    modal: true,
+	    items: [panel],
+	    buttons: [
+		{
+		    xtype: 'button',
+		    iconCls: 'fa fa-clipboard',
+		    handler: function(button) {
+			window.getSelection().selectAllChildren(
+			    document.getElementById('pkgversions'),
+			);
+			document.execCommand("copy");
+		    },
+		    text: gettext('Copy'),
+		},
+		{
+		    text: gettext('Ok'),
+		    handler: function() {
+			this.up('window').close();
+		    },
+		},
+	    ],
+	});
+
+	Proxmox.Utils.API2Request({
+	    waitMsgTarget: me,
+	    url: `/nodes/localhost/apt/versions`,
+	    method: 'GET',
+	    failure: function(response, opts) {
+		win.close();
+		Ext.Msg.alert(gettext('Error'), response.htmlStatus);
+	    },
+	    success: function(response, opts) {
+		let text = '';
+		Ext.Array.each(response.result.data, function(rec) {
+		    let version = "not correctly installed";
+		    let pkg = rec.Package;
+		    if (rec.OldVersion && rec.OldVersion !== 'unknown') {
+			version = rec.OldVersion;
+		    }
+		    if (rec.ExtraInfo) {
+			text += `${pkg}: ${version} (${rec.ExtraInfo})\n`;
+		    } else {
+			text += `${pkg}: ${version}\n`;
+		    }
+		});
+
+		win.show();
+		panel.update(Ext.htmlEncode(text));
+	    },
+	});
+    },
+
     initComponent: function() {
         var me = this;
 
@@ -94,7 +166,15 @@ Ext.define('PBS.ServerStatus', {
 	    },
 	});
 
-	me.tbar = [consoleBtn, restartBtn, shutdownBtn, '->', { xtype: 'proxmoxRRDTypeSelector' }];
+	let version_btn = new Ext.Button({
+	    text: gettext('Package versions'),
+	    iconCls: 'fa fa-gift',
+	    handler: function() {
+		Proxmox.Utils.checked_command(function() { me.showVersions(); });
+	    },
+	});
+
+	me.tbar = [version_btn, '-', consoleBtn, '-', restartBtn, shutdownBtn, '->', { xtype: 'proxmoxRRDTypeSelector' }];
 
 	var rrdstore = Ext.create('Proxmox.data.RRDStore', {
 	    rrdurl: "/api2/json/nodes/localhost/rrd",

www/Subscription.js

@@ -2,13 +2,14 @@ Ext.define('PBS.SubscriptionKeyEdit', {
     extend: 'Proxmox.window.Edit',
 
     title: gettext('Upload Subscription Key'),
-    width: 300,
+    width: 320,
     autoLoad: true,
 
-    onlineHelp: 'getting_help',
+    onlineHelp: 'get_help',
 
     items: {
 	xtype: 'textfield',
+	labelWidth: 120,
 	name: 'key',
 	value: '',
 	fieldLabel: gettext('Subscription Key'),
@@ -22,7 +23,8 @@ Ext.define('PBS.Subscription', {
     title: gettext('Subscription'),
     border: true,
 
-    onlineHelp: 'getting_help',
+    onlineHelp: 'get_help',
+    tools: [PBS.Utils.get_help_tool("get_help")],
 
     viewConfig: {
 	enableTextSelection: true,
@@ -140,16 +142,18 @@ Ext.define('PBS.Subscription', {
 	    tbar: [
 		{
 		    text: gettext('Upload Subscription Key'),
+		    iconCls: 'fa fa-ticket',
 		    handler: function() {
 			let win = Ext.create('PBS.SubscriptionKeyEdit', {
 			    url: '/api2/extjs/' + baseurl,
+			    autoShow: true,
 			});
-			win.show();
 			win.on('destroy', reload);
 		    },
 		},
 		{
 		    text: gettext('Check'),
+		    iconCls: 'fa fa-check-square-o',
 		    handler: function() {
 			Proxmox.Utils.API2Request({
 			    params: { force: 1 },
@@ -171,10 +175,12 @@ Ext.define('PBS.Subscription', {
 		    dangerous: true,
 		    selModel: false,
 		    callback: reload,
+		    iconCls: 'fa fa-trash-o',
 		},
 		'-',
 		{
 		    text: gettext('System Report'),
+		    iconCls: 'fa fa-stethoscope',
 		    handler: function() {
 			Proxmox.Utils.checked_command(function() { me.showReport(); });
 		    },

www/SystemConfiguration.js

@@ -6,6 +6,7 @@ Ext.define('PBS.SystemConfiguration', {
     border: true,
     scrollable: true,
     defaults: { border: false },
+    tools: [PBS.Utils.get_help_tool("sysadmin-network-configuration")],
     items: [
 	{
 	    title: gettext('Network/Time'),

www/Utils.js

@@ -50,6 +50,8 @@ Ext.define('PBS.Utils', {
 	}
     },
 
+    noSubKeyHtml: 'You do not have a valid subscription for this server. Please visit <a target="_blank" href="https://www.proxmox.com/proxmox-backup-server/pricing">www.proxmox.com</a> to get a list of available options.',
+
     getDataStoreFromPath: function(path) {
 	return path.slice(PBS.Utils.dataStorePrefix.length);
     },
@@ -58,6 +60,81 @@ Ext.define('PBS.Utils', {
 	return path.indexOf(PBS.Utils.dataStorePrefix) === 0;
     },
 
+    parsePropertyString: function(value, defaultKey) {
+	var res = {},
+	    error;
+
+	if (typeof value !== 'string' || value === '') {
+	    return res;
+	}
+
+	Ext.Array.each(value.split(','), function(p) {
+	    var kv = p.split('=', 2);
+	    if (Ext.isDefined(kv[1])) {
+		res[kv[0]] = kv[1];
+	    } else if (Ext.isDefined(defaultKey)) {
+		if (Ext.isDefined(res[defaultKey])) {
+		    error = 'defaultKey may be only defined once in propertyString';
+		    return false; // break
+		}
+		res[defaultKey] = kv[0];
+	    } else {
+		error = 'invalid propertyString, not a key=value pair and no defaultKey defined';
+		return false; // break
+	    }
+	    return true;
+	});
+
+	if (error !== undefined) {
+	    console.error(error);
+	    return null;
+	}
+
+	return res;
+    },
+
+    printPropertyString: function(data, defaultKey) {
+	var stringparts = [],
+	    gotDefaultKeyVal = false,
+	    defaultKeyVal;
+
+	Ext.Object.each(data, function(key, value) {
+	    if (defaultKey !== undefined && key === defaultKey) {
+		gotDefaultKeyVal = true;
+		defaultKeyVal = value;
+	    } else if (value !== '' && value !== undefined) {
+		stringparts.push(key + '=' + value);
+	    }
+	});
+
+	stringparts = stringparts.sort();
+	if (gotDefaultKeyVal) {
+	    stringparts.unshift(defaultKeyVal);
+	}
+
+	return stringparts.join(',');
+    },
+
+    // helper for deleting field which are set to there default values
+    delete_if_default: function(values, fieldname, default_val, create) {
+	if (values[fieldname] === '' || values[fieldname] === default_val) {
+	    if (!create) {
+		if (values.delete) {
+		    if (Ext.isArray(values.delete)) {
+			values.delete.push(fieldname);
+		    } else {
+			values.delete += ',' + fieldname;
+		    }
+		} else {
+		    values.delete = [fieldname];
+		}
+	    }
+
+	    delete values[fieldname];
+	}
+    },
+
+
     render_datetime_utc: function(datetime) {
 	let pad = (number) => number < 10 ? '0' + number : number;
 	return datetime.getUTCFullYear() +
@@ -70,7 +147,7 @@ Ext.define('PBS.Utils', {
     },
 
     render_datastore_worker_id: function(id, what) {
-	const res = id.match(/^(\S+?)_(\S+?)_(\S+?)(_(.+))?$/);
+	const res = id.match(/^(\S+?):(\S+?)\/(\S+?)(\/(.+))?$/);
 	if (res) {
 	    let datastore = res[1], backupGroup = `${res[2]}/${res[3]}`;
 	    if (res[4] !== undefined) {
@@ -84,6 +161,34 @@ Ext.define('PBS.Utils', {
 	return `Datastore ${what} ${id}`;
     },
 
+    parse_datastore_worker_id: function(type, id) {
+	let result;
+	let res;
+	if (type.startsWith('verif')) {
+	    res = PBS.Utils.VERIFICATION_JOB_ID_RE.exec(id);
+	    if (res) {
+		result = res[1];
+	    }
+	} else if (type.startsWith('sync')) {
+	    res = PBS.Utils.SYNC_JOB_ID_RE.exec(id);
+	    if (res) {
+		result = res[3];
+	    }
+	} else if (type === 'backup') {
+	    res = PBS.Utils.BACKUP_JOB_ID_RE.exec(id);
+	    if (res) {
+		result = res[1];
+	    }
+	} else if (type === 'garbage_collection') {
+	    return id;
+	} else if (type === 'prune') {
+	    return id;
+	}
+
+
+	return result;
+    },
+
     extractTokenUser: function(tokenid) {
 	return tokenid.match(/^(.+)!([^!]+)$/)[1];
     },
@@ -92,16 +197,82 @@ Ext.define('PBS.Utils', {
 	return tokenid.match(/^(.+)!([^!]+)$/)[2];
     },
 
+    render_estimate: function(value) {
+	if (!value) {
+	    return gettext('Not enough data');
+	}
+
+	let now = new Date();
+	let estimate = new Date(value*1000);
+
+	let timespan = (estimate - now)/1000;
+
+	if (Number(estimate) <= Number(now) || isNaN(timespan)) {
+	    return gettext('Never');
+	}
+
+	let duration = Proxmox.Utils.format_duration_human(timespan);
+	return Ext.String.format(gettext("in {0}"), duration);
+    },
+
+    render_size_usage: function(val, max) {
+	if (max === 0) {
+	    return gettext('N/A');
+	}
+	return (val*100/max).toFixed(2) + '% (' +
+	    Ext.String.format(gettext('{0} of {1}'),
+	    Proxmox.Utils.format_size(val), Proxmox.Utils.format_size(max)) + ')';
+    },
+
+    get_help_tool: function(blockid) {
+	let info = Proxmox.Utils.get_help_info(blockid);
+	if (info === undefined) {
+	    info = Proxmox.Utils.get_help_info('pbs_documentation_index');
+	}
+	if (info === undefined) {
+	    throw "get_help_info failed"; // should not happen
+	}
+
+	let docsURI = window.location.origin + info.link;
+	let title = info.title;
+	if (info.subtitle) {
+	    title += ' - ' + info.subtitle;
+	}
+	return {
+	    type: 'help',
+	    tooltip: title,
+	    handler: function() {
+		window.open(docsURI);
+	    },
+        };
+    },
+
+    calculate_dedup_factor: function(gcstatus) {
+	let dedup = 1.0;
+	if (gcstatus['disk-bytes'] > 0) {
+	    dedup = (gcstatus['index-data-bytes'] || 0)/gcstatus['disk-bytes'];
+	}
+	return dedup;
+    },
+
     constructor: function() {
 	var me = this;
 
+	let PROXMOX_SAFE_ID_REGEX = "([A-Za-z0-9_][A-Za-z0-9._-]*)";
+	// only anchored at beginning
+	// only parses datastore for now
+	me.VERIFICATION_JOB_ID_RE = new RegExp("^" + PROXMOX_SAFE_ID_REGEX + ':?');
+	me.SYNC_JOB_ID_RE = new RegExp("^" + PROXMOX_SAFE_ID_REGEX + ':' +
+	    PROXMOX_SAFE_ID_REGEX + ':' + PROXMOX_SAFE_ID_REGEX + ':');
+	me.BACKUP_JOB_ID_RE = new RegExp("^" + PROXMOX_SAFE_ID_REGEX + ':');
+
 	// do whatever you want here
 	Proxmox.Utils.override_task_descriptions({
 	    backup: (type, id) => PBS.Utils.render_datastore_worker_id(id, gettext('Backup')),
 	    dircreate: [gettext('Directory Storage'), gettext('Create')],
 	    dirremove: [gettext('Directory'), gettext('Remove')],
 	    garbage_collection: ['Datastore', gettext('Garbage collect')],
-	    logrotate: [gettext('Log'), gettext('Rotation')],
+	    logrotate: [null, gettext('Log Rotation')],
 	    prune: (type, id) => PBS.Utils.render_datastore_worker_id(id, gettext('Prune')),
 	    reader: (type, id) => PBS.Utils.render_datastore_worker_id(id, gettext('Read objects')),
 	    sync: ['Datastore', gettext('Remote Sync')],

www/config/ACLView.js

@@ -22,7 +22,12 @@ Ext.define('PBS.config.ACLView', {
 
     title: gettext('Permissions'),
 
+    // Show only those permissions, which can affect this and children paths.
+    // That means that also higher up, "shorter" paths are included, as those
+    // can have a say in the rights on the asked path.
     aclPath: undefined,
+
+    // tell API to only return ACLs matching exactly the aclPath config.
     aclExact: undefined,
 
     controller: {
@@ -83,12 +88,37 @@ Ext.define('PBS.config.ACLView', {
 	    let proxy = view.getStore().rstore.getProxy();
 
 	    let params = {};
-	    if (view.aclPath !== undefined) {
+	    if (typeof view.aclPath === "string") {
-		params.path = view.aclPath;
+		let pathFilter = Ext.create('Ext.util.Filter', {
+		    filterPath: view.aclPath,
+		    filterAtoms: view.aclPath.split('/'),
+		    filterFn: function(item) {
+			let me = this;
+			let path = item.data.path;
+			if (path === "/" || path === me.filterPath) {
+			    return true;
+			} else if (path.length > me.filterPath.length) {
+			    return path.startsWith(me.filterPath + '/');
+			}
+			let pathAtoms = path.split('/');
+			let commonLength = Math.min(pathAtoms.length, me.filterAtoms.length);
+			for (let i = 1; i < commonLength; i++) {
+			    if (me.filterAtoms[i] !== pathAtoms[i]) {
+				return false;
+			    }
+			}
+			return true;
+		    },
+		});
+		view.getStore().addFilter(pathFilter);
 	    }
 	    if (view.aclExact !== undefined) {
+		if (view.aclPath !== undefined) {
+		    params.path = view.aclPath;
+		}
 		params.exact = view.aclExact;
 	    }
+
 	    proxy.setExtraParams(params);
 	    Proxmox.Utils.monStoreErrors(view, view.getStore().rstore);
 	},

www/config/RemoteView.js

@@ -1,6 +1,6 @@
 Ext.define('pmx-remotes', {
     extend: 'Ext.data.Model',
-    fields: ['name', 'host', 'port', 'userid', 'fingerprint', 'comment',
+    fields: ['name', 'host', 'port', 'auth-id', 'fingerprint', 'comment',
 	{
 	    name: 'server',
 	    calculate: function(data) {
@@ -32,6 +32,8 @@ Ext.define('PBS.config.RemoteView', {
 
     title: gettext('Remotes'),
 
+    tools: [PBS.Utils.get_help_tool("backup-remote")],
+
     controller: {
 	xclass: 'Ext.app.ViewController',
 
@@ -127,11 +129,11 @@ Ext.define('PBS.config.RemoteView', {
 	    dataIndex: 'server',
 	},
 	{
-	    header: gettext('User name'),
+	    header: gettext('Auth ID'),
 	    width: 200,
 	    sortable: true,
 	    renderer: Ext.String.htmlEncode,
-	    dataIndex: 'userid',
+	    dataIndex: 'auth-id',
 	},
 	{
 	    header: gettext('Fingerprint'),

www/config/SyncView.js

@@ -162,9 +162,11 @@ Ext.define('PBS.config.SyncJobView', {
 	reload: function() { this.getView().getStore().rstore.load(); },
 
 	init: function(view) {
-	    view.getStore().rstore.getProxy().setExtraParams({
+	    let params = {};
-		store: view.datastore,
+	    if (view.datastore !== undefined) {
-	    });
+		params.store = view.datastore;
+	    }
+	    view.getStore().rstore.getProxy().setExtraParams(params);
 	    Proxmox.Utils.monStoreErrors(view, view.getStore().rstore);
 	},
     },
@@ -237,6 +239,12 @@ Ext.define('PBS.config.SyncJobView', {
 	    flex: 1,
 	    sortable: true,
 	},
+	{
+	    header: gettext('Local Store'),
+	    dataIndex: 'store',
+	    width: 120,
+	    sortable: true,
+	},
 	{
 	    header: gettext('Remote'),
 	    dataIndex: 'remote',
@@ -249,12 +257,6 @@ Ext.define('PBS.config.SyncJobView', {
 	    width: 120,
 	    sortable: true,
 	},
-	{
-	    header: gettext('Local Store'),
-	    dataIndex: 'store',
-	    width: 120,
-	    sortable: true,
-	},
 	{
 	    header: gettext('Owner'),
 	    dataIndex: 'owner',
@@ -304,4 +306,18 @@ Ext.define('PBS.config.SyncJobView', {
 	    sortable: true,
 	},
     ],
+
+    initComponent: function() {
+	let me = this;
+	let hideLocalDatastore = !!me.datastore;
+
+	for (let column of me.columns) {
+	    if (column.dataIndex === 'store') {
+		column.hidden = hideLocalDatastore;
+		break;
+	    }
+	}
+
+	me.callParent();
+    },
 });

www/config/VerifyView.js

@@ -157,9 +157,11 @@ Ext.define('PBS.config.VerifyJobView', {
 	reload: function() { this.getView().getStore().rstore.load(); },
 
 	init: function(view) {
-	    view.getStore().rstore.getProxy().setExtraParams({
+	    let params = {};
-		store: view.datastore,
+	    if (view.datastore !== undefined) {
-	    });
+		params.store = view.datastore;
+	    }
+	    view.getStore().rstore.getProxy().setExtraParams(params);
 	    Proxmox.Utils.monStoreErrors(view, view.getStore().rstore);
 	},
     },
@@ -232,6 +234,11 @@ Ext.define('PBS.config.VerifyJobView', {
 	    flex: 1,
 	    sortable: true,
 	},
+	{
+	    header: gettext('Datastore'),
+	    dataIndex: 'store',
+	    flex: 1,
+	},
 	{
 	    header: gettext('Skip Verified'),
 	    dataIndex: 'ignore-verified',
@@ -288,4 +295,18 @@ Ext.define('PBS.config.VerifyJobView', {
 	    sortable: true,
 	},
     ],
+
+    initComponent: function() {
+	let me = this;
+	let hideDatastore = !!me.datastore;
+
+	for (let column of me.columns) {
+	    if (column.dataIndex === 'store') {
+		column.hidden = hideDatastore;
+		break;
+	    }
+	}
+
+	me.callParent();
+    },
 });

www/css/ext6-pbs.css

@@ -241,3 +241,15 @@ span.snapshot-comment-column {
     font-weight: bold;
     content: "V.";
 }
+
+.x-action-col-icon {
+    margin: 0 1px;
+    font-size: 14px;
+}
+.x-action-col-icon:before, .x-action-col-icon:after {
+    font-size: 14px;
+}
+.x-action-col-icon:hover:before, .x-action-col-icon:hover:after {
+    text-shadow: 1px 1px 1px #AAA;
+    font-weight: 800;
+}

www/dashboard/DataStoreStatistics.js

@@ -47,24 +47,6 @@ Ext.define('PBS.DatastoreStatistics', {
     controller: {
 	xclass: 'Ext.app.ViewController',
 
-	render_estimate: function(value) {
-	    if (!value) {
-		return gettext('Not enough data');
-	    }
-
-	    let now = new Date();
-	    let estimate = new Date(value*1000);
-
-	    let timespan = (estimate - now)/1000;
-
-	    if (Number(estimate) <= Number(now) || isNaN(timespan)) {
-		return gettext('Never');
-	    }
-
-	    let duration = Proxmox.Utils.format_duration_human(timespan);
-	    return Ext.String.format(gettext("in {0}"), duration);
-	},
-
 	init: function(view) {
 	    Proxmox.Utils.monStoreErrors(view, view.getStore().rstore);
 	},
@@ -111,7 +93,7 @@ Ext.define('PBS.DatastoreStatistics', {
 	    text: gettext('Estimated Full'),
 	    dataIndex: 'estimated-full-date',
 	    sortable: true,
-	    renderer: 'render_estimate',
+	    renderer: PBS.Utils.render_estimate,
 	    flex: 1,
 	    minWidth: 130,
 	    maxWidth: 200,