bump version to 0.9.7-1

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "proxmox-backup"
-version = "0.9.4"
+version = "0.9.7"
 authors = ["Dietmar Maurer <dietmar@proxmox.com>"]
 edition = "2018"
 license = "AGPL-3"
@@ -38,7 +38,7 @@ pam-sys = "0.5"
 percent-encoding = "2.1"
 pin-utils = "0.1.0"
 pathpatterns = "0.1.2"
-proxmox = { version = "0.6.0", features = [ "sortable-macro", "api-macro", "websocket" ] }
+proxmox = { version = "0.7.0", features = [ "sortable-macro", "api-macro", "websocket" ] }
 #proxmox = { git = "git://git.proxmox.com/git/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
 #proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro", "websocket" ] }
 proxmox-fuse = "0.1.0"

Makefile

@@ -19,7 +19,8 @@ USR_SBIN := \
 SERVICE_BIN := \
 	proxmox-backup-api \
 	proxmox-backup-banner \
-	proxmox-backup-proxy
+	proxmox-backup-proxy \
+	proxmox-daily-update \
 
 ifeq ($(BUILD_MODE), release)
 CARGO_BUILD_ARGS += --release

debian/changelog

@@ -1,3 +1,103 @@
+rust-proxmox-backup (0.9.7-1) unstable; urgency=medium
+
+  * ui: add remote store selector
+
+  * tools/daemon: fix reload with open connections
+
+  * pxar/create: fix endless loop for shrinking files
+
+  * pxar/create: handle ErrorKind::Interrupted for file reads
+
+  * ui: add action-button for changing backup group owner
+
+  * docs: add interactive prune simulator
+
+  * verify: fix unprivileged verification jobs
+
+  * tasks: allow access to job tasks
+
+  * drop internal 'backup@pam' owner, sync jobs need to set a explicit owner
+
+  * api: datastore: allow to set "verify-new" option over API
+
+  * ui: datastore: add Options tab, allowing one to change per-datastore
+    notification and verify-new options
+
+  * docs: scroll navigation bar to current active section
+
+ -- Proxmox Support Team <support@proxmox.com>  Mon, 09 Nov 2020 07:36:58 +0100
+
+rust-proxmox-backup (0.9.6-1) unstable; urgency=medium
+
+  * fix #3106: improve queueing new incoming connections
+
+  * fix #2870: sync: ensure a updated ticket is used, if available
+
+  * proxy: log if there are too many open connections
+
+  * ui: SyncJobEdit: fix sending 'delete' values on SyncJob creation
+
+  * datastore config: allow to configure who receives job notify emails
+
+  * ui: fix task description for log rotate
+
+  * proxy: also rotate auth.log file
+
+  * ui: add shell panel under administration
+
+  * ui: sync jobs: only set default schedule when creating new jobs and some
+    other small fixes
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 04 Nov 2020 19:12:57 +0100
+
+rust-proxmox-backup (0.9.5-1) unstable; urgency=medium
+
+  * ui: user menu: allow one to change the language while staying logged in
+
+  * proxmox-backup-manager: add subscription commands
+
+  * server/rest: also accept = as token separator
+
+  * privs: allow reading snapshot notes with Datastore.Audit
+
+  * privs: enforce Datastore.Modify|Backup to set backup notes
+
+  * verify: introduce and use new Datastore.Verify privilege
+
+  * docs: add API tokens to documentation
+
+  * ui: various smaller layout and icon improvements
+
+  * api: implement apt pkg cache for caching pending updates
+
+  * api: apt: implement support to send notification email on new updates
+
+  * add daily update and maintenance task
+
+  * fix #2864: add owner option to sync
+
+  * sync: allow sync for non-superusers under special conditions
+
+  * config: support depreacated netmask when parsing interfaces file
+
+  * server: implement access log rotation with re-open via command socket
+
+  * garbage collect: improve index error messages
+
+  * fix #3039: use the same ID regex for info and api
+
+  * ui: administration: allow extensive filtering of the worker task
+
+  * report: add api endpoint and function to generate report
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 03 Nov 2020 17:41:17 +0100
+
+rust-proxmox-backup (0.9.4-2) unstable; urgency=medium
+
+  * make postinst (update) script more resilient
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 29 Oct 2020 20:09:30 +0100
+
 rust-proxmox-backup (0.9.4-1) unstable; urgency=medium
 
   * implement API-token

debian/control

@@ -34,10 +34,10 @@ Build-Depends: debhelper (>= 11),
  librust-pathpatterns-0.1+default-dev (>= 0.1.2-~~),
  librust-percent-encoding-2+default-dev (>= 2.1-~~),
  librust-pin-utils-0.1+default-dev,
- librust-proxmox-0.6+api-macro-dev,
+ librust-proxmox-0.7+api-macro-dev,
- librust-proxmox-0.6+default-dev,
+ librust-proxmox-0.7+default-dev,
- librust-proxmox-0.6+sortable-macro-dev,
+ librust-proxmox-0.7+sortable-macro-dev,
- librust-proxmox-0.6+websocket-dev,
+ librust-proxmox-0.7+websocket-dev,
  librust-proxmox-fuse-0.1+default-dev,
  librust-pxar-0.6+default-dev (>= 0.6.1-~~),
  librust-pxar-0.6+futures-io-dev (>= 0.6.1-~~),

debian/postinst

@@ -15,12 +15,24 @@ case "$1" in
 	fi
 	deb-systemd-invoke $_dh_action proxmox-backup.service proxmox-backup-proxy.service >/dev/null || true
 
-	flock -w 30 /etc/proxmox-backup/.datastore.lck sed -i '/^\s\+verify-schedule /d' /etc/proxmox-backup/datastore.cfg
+	# FIXME: Remove with 1.1
-
+	if test -n "$2"; then
+		if dpkg --compare-versions "$2" 'lt' '0.9.4-1'; then
+			if grep -s -q  -P -e '^\s+verify-schedule ' /etc/proxmox-backup/datastore.cfg; then
+				echo "NOTE: drop all verify schedules from datastore config."
+				echo "You can now add more flexible verify jobs"
+				flock -w 30 /etc/proxmox-backup/.datastore.lck \
+				    sed -i '/^\s\+verify-schedule /d' /etc/proxmox-backup/datastore.cfg || true
+			fi
+		fi
+		if dpkg --compare-versions "$2" 'le' '0.9.5-1'; then
+			chown --quiet backup:backup /var/log/proxmox-backup/api/auth.log || true
+		fi
+	fi
 	# FIXME: Remove in future version once we're sure no broken entries remain in anyone's files
 	if grep -q -e ':termproxy::[^@]\+: ' /var/log/proxmox-backup/tasks/active; then
 	    echo "Fixing up termproxy user id in task log..."
-	    flock -w 30 /var/log/proxmox-backup/tasks/active.lock sed -i 's/:termproxy::\([^@]\+\): /:termproxy::\1@pam: /' /var/log/proxmox-backup/tasks/active
+	    flock -w 30 /var/log/proxmox-backup/tasks/active.lock sed -i 's/:termproxy::\([^@]\+\): /:termproxy::\1@pam: /' /var/log/proxmox-backup/tasks/active || true
 	fi
     ;;
 

debian/prerm

@@ -6,5 +6,6 @@ set -e
 
 # modeled after dh_systemd_start output
 if [ -d /run/systemd/system ] && [ "$1" = remove ]; then
-    deb-systemd-invoke stop 'proxmox-backup-banner.service' 'proxmox-backup-proxy.service' 'proxmox-backup.service' >/dev/null || true
+    deb-systemd-invoke stop 'proxmox-backup-banner.service' 'proxmox-backup-proxy.service' \
+        'proxmox-backup.service' 'proxmox-backup-daily-update.timer' >/dev/null || true
 fi

debian/proxmox-backup-docs.links

@@ -1 +1,2 @@
 /usr/share/doc/proxmox-backup/proxmox-backup.pdf /usr/share/doc/proxmox-backup/html/proxmox-backup.pdf
+/usr/share/javascript/extjs /usr/share/doc/proxmox-backup/html/prune-simulator/extjs

debian/proxmox-backup-server.install

@@ -1,10 +1,13 @@
 etc/proxmox-backup-proxy.service /lib/systemd/system/
 etc/proxmox-backup.service /lib/systemd/system/
 etc/proxmox-backup-banner.service /lib/systemd/system/
+etc/proxmox-backup-daily-update.service /lib/systemd/system/
+etc/proxmox-backup-daily-update.timer /lib/systemd/system/
 etc/pbstest-beta.list /etc/apt/sources.list.d/
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-api
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-proxy
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-banner
+usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-daily-update
 usr/sbin/proxmox-backup-manager
 usr/share/javascript/proxmox-backup/index.hbs
 usr/share/javascript/proxmox-backup/css/ext6-pbs.css

debian/rules

@@ -38,6 +38,7 @@ override_dh_auto_install:
 	    LIBDIR=/usr/lib/$(DEB_HOST_MULTIARCH)
 
 override_dh_installsystemd:
+	dh_installsystemd -pproxmox-backup-server  proxmox-backup-daily-update.timer
 	# note: we start/try-reload-restart services manually in postinst
 	dh_installsystemd --no-start --no-restart-after-upgrade
 

docs/Makefile

@@ -14,6 +14,10 @@ MANUAL_PAGES := 			\
 	proxmox-backup-client.1		\
 	proxmox-backup-manager.1
 
+PRUNE_SIMULATOR_FILES := 					\
+	prune-simulator/index.html				\
+	prune-simulator/documentation.html			\
+	prune-simulator/prune-simulator.js
 
 # Sphinx documentation setup
 SPHINXOPTS    =
@@ -74,10 +78,11 @@ onlinehelpinfo:
 	@echo "Build finished. OnlineHelpInfo.js is in $(BUILDDIR)/scanrefs."
 
 .PHONY: html
-html: ${GENERATED_SYNOPSIS} images/proxmox-logo.svg custom.css conf.py
+html: ${GENERATED_SYNOPSIS} images/proxmox-logo.svg custom.css conf.py ${PRUNE_SIMULATOR_FILES}
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
-	cp images/proxmox-logo.svg $(BUILDDIR)/html/_static/
+	install -m 0644 custom.js custom.css images/proxmox-logo.svg $(BUILDDIR)/html/_static/
-	cp custom.css $(BUILDDIR)/html/_static/
+	install -dm 0755 $(BUILDDIR)/html/prune-simulator
+	install -m 0644 ${PRUNE_SIMULATOR_FILES} $(BUILDDIR)/html/prune-simulator
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 

docs/backup-client.rst

@@ -12,31 +12,31 @@ on the backup server.
 
   [[username@]server[:port]:]datastore
 
-The default value for ``username`` is ``root@pam``.  If no server is specified,
+The default value for ``username`` is ``root@pam``. If no server is specified,
 the default is the local host (``localhost``).
 
 You can specify a port if your backup server is only reachable on a different
 port (e.g. with NAT and port forwarding).
 
-Note that if the server is an IPv6 address, you have to write it with
+Note that if the server is an IPv6 address, you have to write it with square
-square brackets (e.g. [fe80::01]).
+brackets (for example, `[fe80::01]`).
 
-You can pass the repository with the ``--repository`` command
+You can pass the repository with the ``--repository`` command line option, or
-line option, or by setting the ``PBS_REPOSITORY`` environment
+by setting the ``PBS_REPOSITORY`` environment variable.
-variable.
 
 Here some examples of valid repositories and the real values
 
-================================ ============ ================== ===========
+================================ ================== ================== ===========
-Example                          User         Host:Port          Datastore
+Example                          User               Host:Port          Datastore
-================================ ============ ================== ===========
+================================ ================== ================== ===========
-mydatastore                      ``root@pam`` localhost:8007     mydatastore
+mydatastore                      ``root@pam``       localhost:8007     mydatastore
-myhostname:mydatastore           ``root@pam`` myhostname:8007    mydatastore
+myhostname:mydatastore           ``root@pam``       myhostname:8007    mydatastore
-user@pbs@myhostname:mydatastore  ``user@pbs`` myhostname:8007    mydatastore
+user@pbs@myhostname:mydatastore  ``user@pbs``       myhostname:8007    mydatastore
-192.168.55.55:1234:mydatastore   ``root@pam`` 192.168.55.55:1234 mydatastore
+user\@pbs!token@host:store       ``user@pbs!token`` myhostname:8007    mydatastore
-[ff80::51]:mydatastore           ``root@pam`` [ff80::51]:8007    mydatastore
+192.168.55.55:1234:mydatastore   ``root@pam``       192.168.55.55:1234 mydatastore
-[ff80::51]:1234:mydatastore      ``root@pam`` [ff80::51]:1234    mydatastore
+[ff80::51]:mydatastore           ``root@pam``       [ff80::51]:8007    mydatastore
-================================ ============ ================== ===========
+[ff80::51]:1234:mydatastore      ``root@pam``       [ff80::51]:1234    mydatastore
+================================ ================== ================== ===========
 
 Environment Variables
 ---------------------
@@ -45,16 +45,16 @@ Environment Variables
   The default backup repository.
 
 ``PBS_PASSWORD``
-  When set, this value is used for the password required for the
+  When set, this value is used for the password required for the backup server.
-  backup server.
+  You can also set this to a API token secret.
 
 ``PBS_ENCRYPTION_PASSWORD``
   When set, this value is used to access the secret encryption key (if
   protected by password).
 
 ``PBS_FINGERPRINT`` When set, this value is used to verify the server
-  certificate (only used if the system CA certificates cannot
+  certificate (only used if the system CA certificates cannot validate the
-  validate the certificate).
+  certificate).
 
 
 Output Format
@@ -535,6 +535,29 @@ To remove the ticket, issue a logout:
   # proxmox-backup-client logout
 
 
+.. _changing-backup-owner:
+
+Changing the Owner of a Backup Group
+------------------------------------
+
+By default, the owner of a backup group is the user which was used to originally
+create that backup group (or in the case of sync jobs, ``root@pam``). This
+means that if a user ``mike@pbs`` created a backup, another user ``john@pbs``
+can not be used to create backups in that same backup group.  In case you want
+to change the owner of a backup, you can do so with the below command, using a
+user that has ``Datastore.Modify`` privileges on the datastore.
+
+.. code-block:: console
+
+  # proxmox-backup-client change-owner vm/103 john@pbs
+
+This can also be done from within the web interface, by navigating to the
+`Content` section of the datastore that contains the backup group and
+selecting the user icon under the `Actions` column. Common cases for this could
+be to change the owner of a sync job from ``root@pam``, or to repurpose a
+backup group.
+
+
 .. _backup-pruning:
 
 Pruning and Removing Backups

docs/conf.py

@@ -171,6 +171,7 @@ html_theme_options = {
     'extra_nav_links': {
         'Proxmox Homepage': 'https://proxmox.com',
         'PDF': 'proxmox-backup.pdf',
+        'Prune Simulator' : 'prune-simulator/index.html',
     },
 
     'sidebar_width': '320px',
@@ -228,6 +229,10 @@ html_favicon = 'images/favicon.ico'
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+html_js_files = [
+    'custom.js',
+]
+
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.

docs/custom.js

@@ -0,0 +1,7 @@
+window.addEventListener('DOMContentLoaded', (event) => {
+    let activeSection = document.querySelector("a.current");
+    if (activeSection) {
+        // https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollIntoView
+        activeSection.scrollIntoView({ block: 'center' });
+    }
+});

docs/gui.rst

@@ -4,7 +4,7 @@ Graphical User Interface
 Proxmox Backup Server offers an integrated, web-based interface to manage the
 server. This means that you can carry out all administration tasks through your
 web browser, and that you don't have to worry about installing extra management
-tools. The web interface also provides a built in console, so if you prefer the
+tools. The web interface also provides a built-in console, so if you prefer the
 command line or need some extra control, you have this option.
 
 The web interface can be accessed via https://youripaddress:8007. The default
@@ -28,7 +28,6 @@ Login
 -----
 
 .. image:: images/screenshots/pbs-gui-login-window.png
-  :width: 250
   :align: right
   :alt: PBS login window
 
@@ -44,14 +43,13 @@ GUI Overview
 ------------
 
 .. image:: images/screenshots/pbs-gui-dashboard.png
-  :width: 250
   :align: right
   :alt: PBS GUI Dashboard
 
 The Proxmox Backup Server web interface consists of 3 main sections:
 
 * **Header**: At the top. This shows version information, and contains buttons to view
-  documentation, monitor running tasks, and logout.
+  documentation, monitor running tasks, set the language and logout.
 * **Sidebar**: On the left. This contains the configuration options for
   the server.
 * **Configuration Panel**: In the center. This contains the control interface for the
@@ -79,18 +77,17 @@ Configuration
 The Configuration section contains some system configuration options, such as
 time and network configuration. It also contains the following subsections:
 
-* **User Management**: Add users and manage accounts
+* **Access Control**: Add and manage users, API tokens, and the permissions
-* **Permissions**: Manage permissions for various users
+  associated with these items
 * **Remotes**: Add, edit and remove remotes (see :term:`Remote`)
-* **Sync Jobs**: Manage and run sync jobs to remotes
+* **Subscription**: Upload a subscription key, view subscription status and
-* **Subscription**: Upload a subscription key and view subscription status
+  access a text-based system report.
 
 
 Administration
 ^^^^^^^^^^^^^^
 
 .. image:: images/screenshots/pbs-gui-administration-serverstatus.png
-  :width: 250
   :align: right
   :alt: Administration: Server Status overview
 
@@ -105,7 +102,6 @@ tasks and information. These are:
 * **Tasks**: Task history with multiple filter options
 
 .. image:: images/screenshots/pbs-gui-disks.png
-  :width: 250
   :align: right
   :alt: Administration: Disks
 
@@ -120,16 +116,21 @@ The administration menu item also contains a disk management subsection:
 Datastore
 ^^^^^^^^^
 
-.. image:: images/screenshots/pbs-gui-datastore.png
+.. image:: images/screenshots/pbs-gui-datastore-summary.png
-  :width: 250
   :align: right
   :alt: Datastore Configuration
 
-The Datastore section provides an interface for creating and managing
+The Datastore section contains interfaces for creating and managing
-datastores. It contains a subsection for each datastore on the system, in
+datastores. It contains a button to create a new datastore on the server, as
-which you can use the top panel to view:
+well as a subsection for each datastore on the system, in which you can use the
+top panel to view:
 
+* **Summary**: Access a range of datastore usage statistics
 * **Content**: Information on the datastore's backup groups and their respective
   contents
-* **Statistics**: Usage statistics for the datastore
+* **Prune & GC**: Schedule :ref:`pruning <backup-pruning>` and :ref:`garbage
-* **Permissions**: View and manage permissions for the datastore
+  collection <garbage-collection>` operations, and run garbage collection
+  manually
+* **Sync Jobs**: Create, manage and run :ref:`syncjobs` from remote servers
+* **Verify Jobs**: Create, manage and run :ref:`verification` jobs on the
+  datastore

docs/installation.rst

@@ -9,7 +9,7 @@ Debian_ from the provided package repository.
 
 .. include:: package-repositories.rst
 
-Server installation
+Server Installation
 -------------------
 
 The backup server stores the actual backed up data and provides a web based GUI
@@ -52,7 +52,7 @@ It includes the following:
    is used by default and all existing data is removed.
 
 
-Install `Proxmox Backup`_ server on Debian
+Install `Proxmox Backup`_ Server on Debian
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Proxmox ships as a set of Debian packages which can be installed on top of a
@@ -84,11 +84,11 @@ support, and a set of common and useful packages.
   when LVM_ or ZFS_ is used. The network configuration is completely up to you
   as well.
 
-.. note:: You can access the web interface of the Proxmox Backup Server with
+.. Note:: You can access the web interface of the Proxmox Backup Server with
-  your web browser, using HTTPS on port 8007. For example at
+   your web browser, using HTTPS on port 8007. For example at
- ``https://<ip-or-dns-name>:8007``
+   ``https://<ip-or-dns-name>:8007``
 
-Install Proxmox Backup server on `Proxmox VE`_
+Install Proxmox Backup Server on `Proxmox VE`_
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 After configuring the
@@ -104,14 +104,14 @@ After configuring the
    server to store backups. Should the hypervisor server fail, you can
    still access the backups.
 
-.. note::
+.. Note:: You can access the web interface of the Proxmox Backup Server with
-  You can access the web interface of the Proxmox Backup Server with your web
+   your web browser, using HTTPS on port 8007. For example at
-  browser, using HTTPS on port 8007. For example at ``https://<ip-or-dns-name>:8007``
+   ``https://<ip-or-dns-name>:8007``
 
-Client installation
+Client Installation
 -------------------
 
-Install `Proxmox Backup`_ client on Debian
+Install `Proxmox Backup`_ Client on Debian
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Proxmox ships as a set of Debian packages to be installed on

docs/maintenance.rst

@@ -1,13 +1,148 @@
 Maintenance Tasks
 =================
 
+.. _maintenance_pruning:
+
+Pruning
+-------
+
+Prune lets you specify which backup snapshots you want to keep. The
+following retention options are available:
+
+``keep-last <N>``
+  Keep the last ``<N>`` backup snapshots.
+
+``keep-hourly <N>``
+  Keep backups for the last ``<N>`` hours. If there is more than one
+  backup for a single hour, only the latest is kept.
+
+``keep-daily <N>``
+  Keep backups for the last ``<N>`` days. If there is more than one
+  backup for a single day, only the latest is kept.
+
+``keep-weekly <N>``
+  Keep backups for the last ``<N>`` weeks. If there is more than one
+  backup for a single week, only the latest is kept.
+
+  .. note:: Weeks start on Monday and end on Sunday. The software
+     uses the `ISO week date`_ system and handles weeks at
+     the end of the year correctly.
+
+``keep-monthly <N>``
+  Keep backups for the last ``<N>`` months. If there is more than one
+  backup for a single month, only the latest is kept.
+
+``keep-yearly <N>``
+  Keep backups for the last ``<N>`` years. If there is more than one
+  backup for a single year, only the latest is kept.
+
+The retention options are processed in the order given above. Each option
+only covers backups within its time period. The next option does not take care
+of already covered backups. It will only consider older backups.
+
+Unfinished and incomplete backups will be removed by the prune command unless
+they are newer than the last successful backup. In this case, the last failed
+backup is retained.
+
+Prune Simulator
+^^^^^^^^^^^^^^^
+
+You can use the built-in `prune simulator <prune-simulator/index.html>`_
+to explore the effect of different retetion options with various backup
+schedules.
+
+Manual Pruning
+^^^^^^^^^^^^^^
+
+.. image:: images/screenshots/pbs-gui-datastore-content-prune-group.png
+  :target: _images/pbs-gui-datastore-content-prune-group.png
+  :align: right
+  :alt: Prune and garbage collection options
+
+To access pruning functionality for a specific backup group, you can use the
+prune command line option discussed in :ref:`backup-pruning`, or navigate to
+the **Content** tab of the datastore and click the scissors icon in the
+**Actions** column of the relevant backup group.
+
+Prune Schedules
+^^^^^^^^^^^^^^^
+
+To prune on a datastore level, scheduling options can be found under the
+**Prune & GC** tab of the datastore. Here you can set retention settings and
+edit the interval at which pruning takes place.
+
+.. image:: images/screenshots/pbs-gui-datastore-prunegc.png
+  :target: _images/pbs-gui-datastore-prunegc.png
+  :align: right
+  :alt: Prune and garbage collection options
+
+
+.. _maintenance_gc:
+
 Garbage Collection
 ------------------
 
 You can monitor and run :ref:`garbage collection <garbage-collection>` on the
 Proxmox Backup Server using the ``garbage-collection`` subcommand of
-``proxmox-backup-manager``. You can use the ``start`` subcommand to manually start garbage
+``proxmox-backup-manager``. You can use the ``start`` subcommand to manually
-collection on an entire datastore and the ``status`` subcommand to see
+start garbage collection on an entire datastore and the ``status`` subcommand to
-attributes relating to the :ref:`garbage collection <garbage-collection>`.
+see attributes relating to the :ref:`garbage collection <garbage-collection>`.
 
-.. todo:: Add section on verification
+This functionality can also be accessed in the GUI, by navigating to **Prune &
+GC** from the top panel. From here, you can edit the schedule at which garbage
+collection runs and manually start the operation.
+
+
+.. _verification:
+
+Verification
+------------
+
+.. image:: images/screenshots/pbs-gui-datastore-verifyjob-add.png
+  :target: _images/pbs-gui-datastore-verifyjob-add.png
+  :align: right
+  :alt: Adding a verify job
+
+Proxmox Backup offers various verification options to ensure that backup data is
+intact.  Verification is generally carried out through the creation of verify
+jobs. These are scheduled tasks that run verification at a given interval (see
+:ref:`calendar-events`). With these, you can set whether already verified
+snapshots are ignored, as well as set a time period, after which verified jobs
+are checked again. The interface for creating verify jobs can be found under the
+**Verify Jobs** tab of the datastore.
+
+.. Note:: It is recommended that you reverify all backups at least monthly, even
+  if a previous verification was successful. This is becuase physical drives
+  are susceptible to damage over time, which can cause an old, working backup
+  to become corrupted in a process known as `bit rot/data degradation
+  <https://en.wikipedia.org/wiki/Data_degradation>`_. It is good practice to
+  have a regularly recurring (hourly/daily) verification job, which checks new
+  and expired backups, then another weekly/monthly job that will reverify
+  everything. This way, there will be no surprises when it comes to restoring
+  data.
+
+Aside from using verify jobs, you can also run verification manually on entire
+datastores, backup groups, or snapshots. To do this, navigate to the **Content**
+tab of the datastore and either click *Verify All*, or select the *V.* icon from
+the *Actions* column in the table.
+
+.. _maintenance_notification:
+
+Notifications
+-------------
+
+Proxmox Backup Server can send you notification emails about automatically
+scheduled verification, garbage-collection and synchronization tasks results.
+
+By default, notifications are send to the email address configured for the
+`root@pam` user. You can set that user for each datastore.
+
+You can also change the level of notification received per task type, the
+following options are available:
+
+* Always: send a notification for any scheduled task, independent of the
+  outcome
+
+* Errors: send a notification for any scheduled task resulting in an error
+
+* Never: do not send any notification at all

docs/managing-remotes.rst

@@ -59,13 +59,13 @@ Sync Jobs
   :alt: Add a Sync Job
 
 Sync jobs are configured to pull the contents of a datastore on a **Remote** to
-a local datastore. You can manage sync jobs under **Configuration -> Sync Jobs**
+a local datastore. You can manage sync jobs in the web interface, from the
-in the web interface, or using the ``proxmox-backup-manager sync-job`` command.
+**Sync Jobs** tab of the datastore which you'd like to set one up for, or using
-The configuration information for sync jobs is stored at
+the ``proxmox-backup-manager sync-job`` command.  The configuration information
-``/etc/proxmox-backup/sync.cfg``. To create a new sync job, click the add button
+for sync jobs is stored at ``/etc/proxmox-backup/sync.cfg``. To create a new
-in the GUI, or use the ``create`` subcommand. After creating a sync job, you can
+sync job, click the add button in the GUI, or use the ``create`` subcommand.
-either start it manually on the GUI or provide it with a schedule (see
+After creating a sync job, you can either start it manually from the GUI or
-:ref:`calendar-events`) to run regularly.
+provide it with a schedule (see :ref:`calendar-events`) to run regularly.
 
 .. code-block:: console
 
@@ -79,4 +79,17 @@ either start it manually on the GUI or provide it with a schedule (see
   └────────────┴───────┴────────┴──────────────┴───────────┴─────────┘
   # proxmox-backup-manager sync-job remove pbs2-local
 
+For setting up sync jobs, the configuring user needs the following permissions:
 
+#. ``Remote.Read`` on the ``/remote/{remote}/{remote-store}`` path
+#. at least ``Datastore.Backup`` on the local target datastore (``/datastore/{store}``)
+
+If the ``remove-vanished`` option is set, ``Datastore.Prune`` is required on
+the local datastore as well. If the ``owner`` option is not set (defaulting to
+``root@pam``) or set to something other than the configuring user,
+``Datastore.Modify`` is required as well.
+
+.. note:: A sync job can only sync backup groups that the configured remote's
+  user/API token can read. If a remote is configured with a user/API token that
+  only has ``Datastore.Backup`` privileges, only the limited set of accessible
+  snapshots owned by that user/API token can be synced.

docs/prune-simulator/documentation.html

@@ -0,0 +1,102 @@
+<!DOCTYPE html>
+<html>
+<head>
+<style>
+    /* similar to sphinx alabaster theme ones */
+    body {
+        max-width: 90ch;
+        margin-left: 2ch;
+        margin-right: 2ch;
+        line-height: 1.4em;
+        /* avoid the very high contrast of black on white, tone it down a bit */
+        color: #3E4349;
+        hyphens: auto;
+        text-align: left;
+        font-family: 'Open Sans', sans-serif;
+        font-size: 17px;
+    }
+    h1, h2, h3 {
+        font-family: Lato, sans-serif;
+        font-size: 150%;
+        line-height:1.2
+    }
+    tt, code {
+        background-color: #ecf0f3;
+        color: #222;
+    }
+    pre, tt, code {
+        font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace;
+        font-size: 0.9em;
+    }
+    div.note {
+        background-color: #EEE;
+        border: 1px solid #CCC;
+        margin: 10px 0;
+        padding: 0px 20px;
+    }
+    p.note-title {
+        font-weight: bolder;
+        padding: 0;
+        margin: 10px 0 0 0;
+    }
+    div.note > p.last {
+        margin: 5px 0 10px 0;
+    }
+</style>
+</head>
+<body>
+<p>A simulator to experiment with different backup schedules and prune
+options.</p>
+
+<h3>Schedule</h3>
+<p>Select weekdays with the combobox and input hour and minute
+specification separated by a colon, i.e. <code>HOUR:MINUTE</code>. Each of
+<code>HOUR</code> and <code>MINUTE</code> can be either a single value or
+one of the following:</p>
+<ul class="simple">
+<li>a comma-separated list: e.g., <code>01,02,03</code></li>
+<li>a range: e.g., <code>01..10</code></li>
+<li>a repetition: e.g, <code>05/10</code> (means starting at <code>5</code> every <code>10</code>)</li>
+<li>a combination of the above: e.g., <code>01,05..10,12/02</code></li>
+<li>a <code>*</code> for every possible value</li>
+</ul>
+
+<h3>Pruning</h3>
+<p>Prune lets you systematically delete older backups, retaining backups for
+the last given number of time intervals. The following retention options are
+available:</p>
+<dl class="docutils">
+<dt><code class="docutils literal notranslate"><span class="pre">keep-last</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> backup snapshots.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-hourly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> hours. If there is more than one
+backup for a single hour, only the latest is kept.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-daily</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> days. If there is more than one
+backup for a single day, only the latest is kept.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-weekly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> weeks. If there is more than one
+backup for a single week, only the latest is kept.
+<div class="last admonition note">
+<p class="note-title">Note:</p>
+<p class="last">Weeks start on Monday and end on Sunday. The software
+uses the <a class="reference external" href="https://en.wikipedia.org/wiki/ISO_week_date">ISO week date</a> system and handles weeks at
+the end of the year correctly.</p>
+</div>
+</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-monthly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> months. If there is more than one
+backup for a single month, only the latest is kept.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-yearly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> years. If there is more than one
+backup for a single year, only the latest is kept.</dd>
+</dl>
+<p>The retention options are processed in the order given above. Each option
+only covers backups within its time period. The next option does not take care
+of already covered backups. It will only consider older backups.</p>
+<p>For example, in a week covered by <code>keep-weekly</code>, one backup is
+kept while all others are removed; <code>keep-monthly</code> then does not
+consider backups from that week anymore, even if part of the week is part of
+an earlier month.</p>
+</body>
+</html>

docs/prune-simulator/index.html

@@ -0,0 +1,41 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
+    <title>PBS Prune Simulator</title>
+
+    <link rel="stylesheet" type="text/css" href="extjs/theme-crisp/resources/theme-crisp-all.css">
+    <style>
+        .cal {
+            margin: 5px;
+        }
+        .cal-day {
+            vertical-align: top;
+            width: 150px;
+            border: #939393 1px solid;
+            color: #454545;
+        }
+        .cal-day-date {
+            border-bottom: #444 1px solid;
+            color: #000;
+        }
+        .strikethrough {
+            text-decoration: line-through;
+        }
+        .black {
+            color: #000;
+        }
+        .sun {
+            background-color: #ededed;
+        }
+        .first-of-month {
+            border-right: dashed black 4px;
+        }
+    </style>
+
+    <script type="text/javascript" src="extjs/ext-all.js"></script>
+    <script type="text/javascript" src="prune-simulator.js"></script>
+</head>
+<body></body>
+</html>

docs/prune-simulator/prune-simulator.js

@@ -0,0 +1,773 @@
+// FIXME: HACK! Makes scrolling in number spinner work again. fixed in ExtJS >= 6.1
+if (Ext.isFirefox) {
+    Ext.$eventNameMap.DOMMouseScroll = 'DOMMouseScroll';
+}
+
+Ext.onReady(function() {
+    const NOW = new Date();
+    const COLORS = {
+	'keep-last': 'orange',
+	'keep-hourly': 'purple',
+	'keep-daily': 'yellow',
+	'keep-weekly': 'green',
+	'keep-monthly': 'blue',
+	'keep-yearly': 'red',
+	'all zero': 'white',
+    };
+    const TEXT_COLORS = {
+	'keep-last': 'black',
+	'keep-hourly': 'white',
+	'keep-daily': 'black',
+	'keep-weekly': 'white',
+	'keep-monthly': 'white',
+	'keep-yearly': 'white',
+	'all zero': 'black',
+    };
+
+    Ext.define('PBS.prunesimulator.Documentation', {
+	extend: 'Ext.Panel',
+	alias: 'widget.prunesimulatorDocumentation',
+
+	html: '<iframe style="width:100%;height:100%;border:0px;" src="./documentation.html"/>',
+    });
+
+    Ext.define('PBS.prunesimulator.CalendarEvent', {
+	extend: 'Ext.form.field.ComboBox',
+	alias: 'widget.prunesimulatorCalendarEvent',
+
+	editable: true,
+
+	displayField: 'text',
+	valueField: 'value',
+	queryMode: 'local',
+
+	store: {
+	    field: ['value', 'text'],
+	    data: [
+		{ value: '0/2:00', text: "Every two hours" },
+		{ value: '0/6:00', text: "Every six hours" },
+		{ value: '2,22:30', text: "At 02:30 and 22:30" },
+		{ value: '00:00', text: "At 00:00" },
+		{ value: '08..17:00/30', text: "From 08:00 to 17:30 every 30 minutes" },
+		{ value: 'HOUR:MINUTE', text: "Custom schedule" },
+	    ],
+	},
+
+	tpl: [
+	    '<ul class="x-list-plain"><tpl for=".">',
+	    '<li role="option" class="x-boundlist-item">{text}</li>',
+	    '</tpl></ul>',
+	],
+
+	displayTpl: [
+	    '<tpl for=".">',
+	    '{value}',
+	    '</tpl>',
+	],
+    });
+
+    Ext.define('PBS.prunesimulator.DayOfWeekSelector', {
+	extend: 'Ext.form.field.ComboBox',
+	alias: 'widget.prunesimulatorDayOfWeekSelector',
+
+	editable: false,
+
+	displayField: 'text',
+	valueField: 'value',
+	queryMode: 'local',
+
+	store: {
+	    field: ['value', 'text'],
+	    data: [
+		{ value: 'mon', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[1]) },
+		{ value: 'tue', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[2]) },
+		{ value: 'wed', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[3]) },
+		{ value: 'thu', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[4]) },
+		{ value: 'fri', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[5]) },
+		{ value: 'sat', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[6]) },
+		{ value: 'sun', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[0]) },
+	    ],
+	},
+    });
+
+    Ext.define('pbs-prune-list', {
+	extend: 'Ext.data.Model',
+	fields: [
+	    {
+		name: 'backuptime',
+		type: 'date',
+		dateFormat: 'timestamp',
+	    },
+	    {
+		name: 'mark',
+		type: 'string',
+	    },
+	    {
+		name: 'keepName',
+		type: 'string',
+	    },
+	],
+    });
+
+    Ext.define('PBS.prunesimulator.PruneList', {
+	extend: 'Ext.panel.Panel',
+	alias: 'widget.prunesimulatorPruneList',
+
+	initComponent: function() {
+	    let me = this;
+
+	    if (!me.store) {
+		throw "no store specified";
+	    }
+
+	    me.items = [
+		{
+		    xtype: 'grid',
+		    store: me.store,
+		    border: false,
+		    columns: [
+			{
+			    header: 'Backup Time',
+			    dataIndex: 'backuptime',
+			    renderer: function(value, metaData, record) {
+				let text = Ext.Date.format(value, 'Y-m-d H:i:s');
+				if (record.data.mark === 'keep') {
+				    if (me.useColors) {
+					let bgColor = COLORS[record.data.keepName];
+					let textColor = TEXT_COLORS[record.data.keepName];
+					return '<div style="background-color: ' + bgColor + '; ' +
+							    'color: ' + textColor + ';">' + text + '</div>';
+				    } else {
+					return text;
+				    }
+				} else {
+				    return '<div style="text-decoration: line-through;">' + text + '</div>';
+				}
+			    },
+			    flex: 1,
+			    sortable: false,
+			},
+			{
+			    header: 'Keep (reason)',
+			    dataIndex: 'mark',
+			    renderer: function(value, metaData, record) {
+				if (record.data.mark === 'keep') {
+				    return 'keep (' + record.data.keepName + ')';
+				} else {
+				    return value;
+				}
+			    },
+			    width: 200,
+			    sortable: false,
+			},
+		    ],
+		},
+	    ];
+
+	    me.callParent();
+	},
+    });
+
+    Ext.define('PBS.prunesimulator.WeekTable', {
+	extend: 'Ext.panel.Panel',
+	alias: 'widget.prunesimulatorWeekTable',
+
+	reload: function() {
+	    let me = this;
+	    let backups = me.store.data.items;
+
+	    let html = '<table class="cal">';
+
+	    let now = new Date(NOW.getTime());
+	    let skip = 7 - parseInt(Ext.Date.format(now, 'N'), 10);
+	    let tableStartDate = Ext.Date.add(now, Ext.Date.DAY, skip);
+
+	    let bIndex = 0;
+
+	    for (let i = 0; bIndex < backups.length; i++) {
+		html += '<tr>';
+
+		for (let j = 0; j < 7; j++) {
+		    let date = Ext.Date.subtract(tableStartDate, Ext.Date.DAY, j + 7 * i);
+		    let currentDay = Ext.Date.format(date, 'd/m/Y');
+
+		    let dayOfWeekCls = Ext.Date.format(date, 'D').toLowerCase();
+		    let firstOfMonthCls = Ext.Date.format(date, 'd') === '01'
+		        ? 'first-of-month'
+		        : '';
+		    html += `<td class="cal-day ${dayOfWeekCls} ${firstOfMonthCls}">`;
+
+		    const isBackupOnDay = function(backup, day) {
+			return backup && Ext.Date.format(backup.data.backuptime, 'd/m/Y') === day;
+		    };
+
+		    let backup = backups[bIndex];
+
+		    html += '<table><tr>';
+		    html += `<th class="cal-day-date">${Ext.Date.format(date, 'D, d M Y')}</th>`;
+
+		    while (isBackupOnDay(backup, currentDay)) {
+			html += '<tr><td>';
+
+			let text = Ext.Date.format(backup.data.backuptime, 'H:i');
+			if (backup.data.mark === 'remove') {
+			    html += `<span class="strikethrough">${text}</span>`;
+			} else {
+			    text += ` (${backup.data.keepName})`;
+			    if (me.useColors) {
+				let bgColor = COLORS[backup.data.keepName];
+				let textColor = TEXT_COLORS[backup.data.keepName];
+				html += `<span style="background-color: ${bgColor};
+				    color: ${textColor};">${text}</span>`;
+			    } else {
+				html += `<span class="black">${text}</span>`;
+			    }
+			}
+			html += '</td></tr>';
+			backup = backups[++bIndex];
+		    }
+		    html += '</table>';
+		    html += '</div>';
+		    html += '</td>';
+		}
+
+		html += '</tr>';
+	    }
+
+	    me.setHtml(html);
+	},
+
+	initComponent: function() {
+	    let me = this;
+
+	    if (!me.store) {
+		throw "no store specified";
+	    }
+
+	    let reload = function() {
+		me.reload();
+	    };
+
+	    me.store.on("datachanged", reload);
+
+	    me.callParent();
+
+	    me.reload();
+	},
+    });
+
+    Ext.define('PBS.PruneSimulatorPanel', {
+	extend: 'Ext.panel.Panel',
+	alias: 'widget.prunesimulatorPanel',
+
+	viewModel: {
+	},
+
+	getValues: function() {
+	    let me = this;
+
+	    let values = {};
+
+	    Ext.Array.each(me.query('[isFormField]'), function(field) {
+		let data = field.getSubmitData();
+		Ext.Object.each(data, function(name, val) {
+		    values[name] = val;
+		});
+	    });
+
+	    return values;
+	},
+
+	controller: {
+	    xclass: 'Ext.app.ViewController',
+
+	    init: function(view) {
+		this.reloadFull(); // initial load
+		this.switchColor(true);
+	    },
+
+	    control: {
+		'field[fieldGroup=keep]': { change: 'reloadPrune' },
+	    },
+
+	    reloadFull: function() {
+		let me = this;
+		let view = me.getView();
+
+		let params = view.getValues();
+
+		let [hourSpec, minuteSpec] = params['schedule-time'].split(':');
+
+		if (!hourSpec || !minuteSpec) {
+		    Ext.Msg.alert('Error', 'Invalid schedule');
+		    return;
+		}
+
+		let matchTimeSpec = function(timeSpec, rangeMin, rangeMax) {
+		    let specValues = timeSpec.split(',');
+		    let matches = {};
+
+		    let assertValid = function(value) {
+			let num = Number(value);
+			if (isNaN(num)) {
+			    throw value + " is not an integer";
+			} else if (value < rangeMin || value > rangeMax) {
+			    throw "number '" + value + "' is not in the range '" + rangeMin + ".." + rangeMax + "'";
+			}
+			return num;
+		    };
+
+		    specValues.forEach(function(value) {
+			if (value.includes('..')) {
+			    let [start, end] = value.split('..');
+			    start = assertValid(start);
+			    end = assertValid(end);
+			    if (start > end) {
+				throw "interval start is bigger then interval end '" + start + " > " + end + "'";
+			    }
+			    for (let i = start; i <= end; i++) {
+				matches[i] = 1;
+			    }
+			} else if (value.includes('/')) {
+			    let [start, step] = value.split('/');
+			    start = assertValid(start);
+			    step = assertValid(step);
+			    for (let i = start; i <= rangeMax; i += step) {
+				matches[i] = 1;
+			    }
+			} else if (value === '*') {
+			    for (let i = rangeMin; i <= rangeMax; i++) {
+				matches[i] = 1;
+			    }
+			} else {
+			    value = assertValid(value);
+			    matches[value] = 1;
+			}
+		    });
+
+		    return Object.keys(matches);
+		};
+
+		let hours, minutes;
+
+		try {
+		    hours = matchTimeSpec(hourSpec, 0, 23);
+		    minutes = matchTimeSpec(minuteSpec, 0, 59);
+		} catch (err) {
+		    Ext.Msg.alert('Error', err);
+		    return;
+		}
+
+		let backups = me.populateFromSchedule(
+		    params['schedule-weekdays'],
+		    hours,
+		    minutes,
+		    params.numberOfWeeks,
+		);
+
+		me.pruneSelect(backups, params);
+
+		view.pruneStore.setData(backups);
+	    },
+
+	    reloadPrune: function() {
+		let me = this;
+		let view = me.getView();
+
+		let params = view.getValues();
+
+		let backups = [];
+		view.pruneStore.getData().items.forEach(function(item) {
+		    backups.push({
+			backuptime: item.data.backuptime,
+		    });
+		});
+
+		me.pruneSelect(backups, params);
+
+		view.pruneStore.setData(backups);
+	    },
+
+	    // backups are sorted descending by date
+	    populateFromSchedule: function(weekdays, hours, minutes, weekCount) {
+		let weekdayFlags = [
+		    weekdays.includes('sun'),
+		    weekdays.includes('mon'),
+		    weekdays.includes('tue'),
+		    weekdays.includes('wed'),
+		    weekdays.includes('thu'),
+		    weekdays.includes('fri'),
+		    weekdays.includes('sat'),
+		];
+
+		let todaysDate = new Date(NOW.getTime());
+
+		let timesOnSingleDay = [];
+
+		hours.forEach(function(hour) {
+		    minutes.forEach(function(minute) {
+			todaysDate.setHours(hour);
+			todaysDate.setMinutes(minute);
+			timesOnSingleDay.push(todaysDate.getTime());
+		    });
+		});
+
+		// ordering here and iterating backwards through days
+		// ensures that everything is ordered
+		timesOnSingleDay.sort(function(a, b) {
+		    return a < b;
+		});
+
+		let backups = [];
+
+		for (let i = 0; i < 7 * weekCount; i++) {
+		    let daysDate = Ext.Date.subtract(todaysDate, Ext.Date.DAY, i);
+		    let weekday = parseInt(Ext.Date.format(daysDate, 'w'), 10);
+		    if (weekdayFlags[weekday]) {
+			timesOnSingleDay.forEach(function(time) {
+			    backups.push({
+				backuptime: Ext.Date.subtract(new Date(time), Ext.Date.DAY, i),
+			    });
+			});
+		    }
+		}
+
+		return backups;
+	    },
+
+	    pruneMark: function(backups, keepCount, keepName, idFunc) {
+		if (!keepCount) {
+		    return;
+		}
+
+		let alreadyIncluded = {};
+		let newlyIncluded = {};
+		let newlyIncludedCount = 0;
+
+		let finished = false;
+
+		backups.forEach(function(backup) {
+		    let mark = backup.mark;
+		    let id = idFunc(backup);
+
+		    if (finished || alreadyIncluded[id]) {
+			return;
+		    }
+
+		    if (mark) {
+			if (mark === 'keep') {
+			    alreadyIncluded[id] = true;
+			}
+			return;
+		    }
+
+		    if (!newlyIncluded[id]) {
+			if (newlyIncludedCount >= keepCount) {
+			    finished = true;
+			    return;
+			}
+			newlyIncluded[id] = true;
+			newlyIncludedCount++;
+			backup.mark = 'keep';
+			backup.keepName = keepName;
+		    } else {
+			backup.mark = 'remove';
+		    }
+		});
+	    },
+
+	    // backups need to be sorted descending by date
+	    pruneSelect: function(backups, keepParams) {
+		let me = this;
+
+		if (Number(keepParams['keep-last']) +
+		    Number(keepParams['keep-hourly']) +
+		    Number(keepParams['keep-daily']) +
+		    Number(keepParams['keep-weekly']) +
+		    Number(keepParams['keep-monthly']) +
+		    Number(keepParams['keep-yearly']) === 0) {
+		    backups.forEach(function(backup) {
+			backup.mark = 'keep';
+			backup.keepName = 'all zero';
+		    });
+
+		    return;
+		}
+
+		me.pruneMark(backups, keepParams['keep-last'], 'keep-last', function(backup) {
+		    return backup.backuptime;
+		});
+		me.pruneMark(backups, keepParams['keep-hourly'], 'keep-hourly', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'H/d/m/Y');
+		});
+		me.pruneMark(backups, keepParams['keep-daily'], 'keep-daily', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'd/m/Y');
+		});
+		me.pruneMark(backups, keepParams['keep-weekly'], 'keep-weekly', function(backup) {
+		    // ISO-8601 week and week-based year
+		    return Ext.Date.format(backup.backuptime, 'W/o');
+		});
+		me.pruneMark(backups, keepParams['keep-monthly'], 'keep-monthly', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'm/Y');
+		});
+		me.pruneMark(backups, keepParams['keep-yearly'], 'keep-yearly', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'Y');
+		});
+
+		backups.forEach(function(backup) {
+		    backup.mark = backup.mark || 'remove';
+		});
+	    },
+
+	    toggleColors: function(checkbox, checked) {
+		this.switchColor(checked);
+	    },
+
+	    switchColor: function(useColors) {
+		let me = this;
+		let view = me.getView();
+
+		const getStyle = name =>
+		    `background-color: ${COLORS[name]}; color: ${TEXT_COLORS[name]};`;
+
+		for (const field of view.query('[isFormField]')) {
+		    if (field.fieldGroup !== 'keep') {
+			continue;
+		    }
+		    if (useColors) {
+		    field.setFieldStyle(getStyle(field.name));
+		    } else {
+			field.setFieldStyle('background-color: white; color: #444;');
+		    }
+		}
+
+		me.lookup('weekTable').useColors = useColors;
+		me.lookup('pruneList').useColors = useColors;
+
+		me.reloadPrune();
+	    },
+	},
+
+	keepItems: [
+	    {
+		xtype: 'numberfield',
+		name: 'keep-last',
+		allowBlank: true,
+		fieldLabel: 'keep-last',
+		minValue: 0,
+		value: 4,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-hourly',
+		allowBlank: true,
+		fieldLabel: 'keep-hourly',
+		minValue: 0,
+		value: 0,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-daily',
+		allowBlank: true,
+		fieldLabel: 'keep-daily',
+		minValue: 0,
+		value: 5,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-weekly',
+		allowBlank: true,
+		fieldLabel: 'keep-weekly',
+		minValue: 0,
+		value: 2,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-monthly',
+		allowBlank: true,
+		fieldLabel: 'keep-monthly',
+		minValue: 0,
+		value: 0,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-yearly',
+		allowBlank: true,
+		fieldLabel: 'keep-yearly',
+		minValue: 0,
+		value: 0,
+		fieldGroup: 'keep',
+	    },
+	],
+
+	initComponent: function() {
+	    var me = this;
+
+	    me.pruneStore = Ext.create('Ext.data.Store', {
+		model: 'pbs-prune-list',
+		sorters: { property: 'backuptime', direction: 'DESC' },
+	    });
+
+	    let scheduleItems = [
+		{
+		    xtype: 'prunesimulatorDayOfWeekSelector',
+		    name: 'schedule-weekdays',
+		    fieldLabel: 'Day of week',
+		    value: ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'],
+		    allowBlank: false,
+		    multiSelect: true,
+		    padding: '0 0 0 10',
+		},
+		{
+		    xtype: 'prunesimulatorCalendarEvent',
+		    name: 'schedule-time',
+		    allowBlank: false,
+		    value: '0/6:00',
+		    fieldLabel: 'Backup schedule',
+		    padding: '0 0 0 10',
+		},
+		{
+		    xtype: 'numberfield',
+		    name: 'numberOfWeeks',
+		    allowBlank: false,
+		    fieldLabel: 'Number of weeks',
+		    minValue: 1,
+		    value: 15,
+		    maxValue: 260, // five years
+		    padding: '0 0 0 10',
+		},
+		{
+		    xtype: 'button',
+		    name: 'schedule-button',
+		    text: 'Update Schedule',
+		    handler: 'reloadFull',
+		},
+	    ];
+
+	    me.items = [
+		{
+		    xtype: 'panel',
+		    layout: {
+			type: 'hbox',
+			align: 'stretch',
+		    },
+		    border: false,
+		    items: [
+			{
+			    title: 'View',
+			    layout: 'anchor',
+			    flex: 1,
+			    border: false,
+			    bodyPadding: 10,
+			    items: [
+				{
+				    xtype: 'checkbox',
+				    name: 'showCalendar',
+				    reference: 'showCalendar',
+				    fieldLabel: 'Show Calendar:',
+				    checked: true,
+				},
+				{
+				    xtype: 'checkbox',
+				    name: 'showColors',
+				    reference: 'showColors',
+				    fieldLabel: 'Show Colors:',
+				    checked: true,
+				    handler: 'toggleColors',
+				},
+			    ],
+			},
+			{ xtype: "panel", width: 1, border: 1 },
+			{
+			    layout: 'anchor',
+			    flex: 1,
+			    border: false,
+			    title: 'Simulated Backup Schedule',
+			    defaults: {
+				labelWidth: 120,
+			    },
+			    bodyPadding: 10,
+			    items: scheduleItems,
+			},
+		    ],
+		},
+		{
+		    xtype: 'panel',
+		    layout: {
+			type: 'hbox',
+			align: 'stretch',
+		    },
+		    flex: 1,
+		    border: false,
+		    items: [
+			{
+			    layout: 'anchor',
+			    title: 'Prune Options',
+			    border: false,
+			    bodyPadding: 10,
+			    scrollable: true,
+			    items: me.keepItems,
+			    flex: 1,
+			},
+			{ xtype: "panel", width: 1, border: 1 },
+			{
+			    layout: 'fit',
+			    title: 'Backups',
+			    border: false,
+			    xtype: 'prunesimulatorPruneList',
+			    store: me.pruneStore,
+			    reference: 'pruneList',
+			    flex: 1,
+			},
+		    ],
+		},
+		{
+		    layout: 'anchor',
+		    title: 'Calendar',
+		    autoScroll: true,
+		    flex: 2,
+		    xtype: 'prunesimulatorWeekTable',
+		    reference: 'weekTable',
+		    store: me.pruneStore,
+		    bind: {
+			hidden: '{!showCalendar.checked}',
+		    },
+		},
+	    ];
+
+	    me.callParent();
+	},
+    });
+
+    Ext.create('Ext.container.Viewport', {
+	layout: 'border',
+	renderTo: Ext.getBody(),
+	items: [
+	    {
+		xtype: 'prunesimulatorPanel',
+		title: 'Proxmox Backup Server - Prune Simulator',
+		region: 'west',
+		layout: {
+		    type: 'vbox',
+		    align: 'stretch',
+		    pack: 'start',
+		},
+		flex: 3,
+		maxWidth: 1090,
+	    },
+	    {
+		xtype: 'prunesimulatorDocumentation',
+		title: 'Usage',
+		border: false,
+		flex: 2,
+		region: 'center',
+	    },
+	],
+    });
+});
+

docs/storage.rst

@@ -107,7 +107,7 @@ is stored in the file ``/etc/proxmox-backup/datastore.cfg``.
 Datastore Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-.. image:: images/screenshots/pbs-gui-datastore.png
+.. image:: images/screenshots/pbs-gui-datastore-content.png
   :align: right
   :alt: Datastore Overview
 
@@ -127,8 +127,9 @@ Creating a Datastore
   :align: right
   :alt: Create a datastore
 
-You can create a new datastore from the web GUI, by navigating to **Datastore** in
+You can create a new datastore from the web interface, by clicking **Add
-the menu tree and clicking **Create**. Here:
+Datastore** in the side menu, under the **Datastore** section. In the setup
+window:
 
 * *Name* refers to the name of the datastore
 * *Backing Path* is the path to the directory upon which you want to create the
@@ -136,7 +137,9 @@ the menu tree and clicking **Create**. Here:
 * *GC Schedule* refers to the time and intervals at which garbage collection
   runs
 * *Prune Schedule* refers to the frequency at which pruning takes place
-* *Prune Options* set the amount of backups which you would like to keep (see :ref:`backup-pruning`).
+* *Prune Options* set the amount of backups which you would like to keep (see
+  :ref:`backup-pruning`).
+* *Comment* can be used to add some contextual information to the datastore.
 
 Alternatively you can create a new datastore from the command line. The
 following command creates a new datastore called ``store1`` on :file:`/backup/disk1/store1`

docs/user-management.rst

@@ -41,11 +41,12 @@ users:
   :alt: Add a new user
 
 The superuser has full administration rights on everything, so you
-normally want to add other users with less privileges. You can create a new
+normally want to add other users with less privileges. You can add a new
-user with the ``user create`` subcommand or through the web interface, under
+user with the ``user create`` subcommand or through the web
-**Configuration -> User Management**. The ``create`` subcommand lets you specify
+interface, under the **User Management** tab of **Configuration -> Access
-many options like ``--email`` or ``--password``. You can update or change any
+Control**. The ``create`` subcommand lets you specify many options like
-user properties using the ``update`` subcommand later (**Edit** in the GUI):
+``--email`` or ``--password``. You can update or change any user properties
+using the ``update`` subcommand later (**Edit** in the GUI):
 
 
 .. code-block:: console
@@ -70,7 +71,7 @@ The resulting user list looks like this:
   │ root@pam │      1 │        │           │          │                  │ Superuser        │
   └──────────┴────────┴────────┴───────────┴──────────┴──────────────────┴──────────────────┘
 
-Newly created users do not have any permissions. Please read the next
+Newly created users do not have any permissions. Please read the Access Control
 section to learn how to set access permissions.
 
 If you want to disable a user account, you can do that by setting ``--enable`` to ``0``
@@ -85,15 +86,77 @@ Or completely remove the user with:
 
   # proxmox-backup-manager user remove john@pbs
 
+.. _user_tokens:
+
+API Tokens
+----------
+
+.. image:: images/screenshots/pbs-gui-apitoken-overview.png
+  :align: right
+  :alt: API Token Overview
+
+Any authenticated user can generate API tokens which can in turn be used to
+configure various clients, instead of directly providing the username and
+password.
+
+API tokens serve two purposes:
+
+#. Easy revocation in case client gets compromised
+#. Limit permissions for each client/token within the users' permission
+
+An API token consists of two parts: an identifier consisting of the user name,
+the realm and a tokenname (``user@realm!tokenname``), and a secret value. Both
+need to be provided to the client in place of the user ID (``user@realm``) and
+the user password, respectively.
+
+.. image:: images/screenshots/pbs-gui-apitoken-secret-value.png
+  :align: right
+  :alt: API secret value
+
+The API token is passed from the client to the server by setting the
+``Authorization`` HTTP header with method ``PBSAPIToken`` to the value
+``TOKENID:TOKENSECRET``.
+
+Generating new tokens can done using ``proxmox-backup-manager`` or the GUI:
+
+.. code-block:: console
+
+  # proxmox-backup-manager user generate-token john@pbs client1
+  Result: {
+    "tokenid": "john@pbs!client1",
+    "value": "d63e505a-e3ec-449a-9bc7-1da610d4ccde"
+  }
+
+.. note:: The displayed secret value needs to be saved, since it cannot be
+  displayed again after generating the API token.
+
+The ``user list-tokens`` sub-command can be used to display tokens and their
+metadata:
+
+.. code-block:: console
+
+  # proxmox-backup-manager user list-tokens john@pbs
+  ┌──────────────────┬────────┬────────┬─────────┐
+  │ tokenid          │ enable │ expire │ comment │
+  ╞══════════════════╪════════╪════════╪═════════╡
+  │ john@pbs!client1 │      1 │        │         │
+  └──────────────────┴────────┴────────┴─────────┘
+
+Similarly, the ``user delete-token`` subcommand can be used to delete a token
+again.
+
+Newly generated API tokens don't have any permissions. Please read the next
+section to learn how to set access permissions.
+
 
 .. _user_acl:
 
 Access Control
 --------------
 
-By default new users do not have any permission. Instead you need to
+By default new users and API tokens do not have any permission. Instead you
-specify what is allowed and what is not. You can do this by assigning
+need to specify what is allowed and what is not. You can do this by assigning
-roles to users on specific objects like datastores or remotes. The
+roles to users/tokens on specific objects like datastores or remotes. The
 following roles exist:
 
 **NoAccess**
@@ -130,7 +193,7 @@ following roles exist:
 **RemoteSyncOperator**
   Is allowed to read data from a remote.
 
-.. image:: images/screenshots/pbs-gui-permissions-add.png
+.. image:: images/screenshots/pbs-gui-user-management-add-user.png
   :align: right
   :alt: Add permissions for user
 
@@ -148,20 +211,21 @@ The data represented in each field is as follows:
 #. The object on which the permission is set. This can be a specific object
    (single datastore, remote, etc.) or a top level object, which with
    propagation enabled, represents all children of the object also.
-#. The user for which the permission is set
+#. The user(s)/token(s) for which the permission is set
 #. The role being set
 
-You can manage datastore permissions from **Configuration -> Permissions** in the
+You can manage permissions via **Configuration -> Access Control ->
-web interface. Likewise, you can use the ``acl`` subcommand to manage and
+Permissions** in the web interface. Likewise, you can use the ``acl``
-monitor user permissions from the command line. For example, the command below
+subcommand to manage and monitor user permissions from the command line. For
-will add the user ``john@pbs`` as a **DatastoreAdmin** for the datastore
+example, the command below will add the user ``john@pbs`` as a
-``store1``, located at ``/backup/disk1/store1``:
+**DatastoreAdmin** for the datastore ``store1``, located at
+``/backup/disk1/store1``:
 
 .. code-block:: console
 
-  # proxmox-backup-manager acl update /datastore/store1 DatastoreAdmin --userid john@pbs
+  # proxmox-backup-manager acl update /datastore/store1 DatastoreAdmin --auth-id john@pbs
 
-You can monitor the roles of each user using the following command:
+You can list the ACLs of each user/token using the following command:
 
 .. code-block:: console
 
@@ -172,7 +236,7 @@ You can monitor the roles of each user using the following command:
    │ john@pbs │ /datastore/disk1 │         1 │ DatastoreAdmin │
    └──────────┴──────────────────┴───────────┴────────────────┘
 
-A single user can be assigned multiple permission sets for different datastores.
+A single user/token can be assigned multiple permission sets for different datastores.
 
 .. Note::
   Naming convention is important here. For datastores on the host,
@@ -183,4 +247,39 @@ A single user can be assigned multiple permission sets for different datastores.
   remote (see `Remote` below) and ``{storename}`` is the name of the datastore on
   the remote.
 
+API Token permissions
+~~~~~~~~~~~~~~~~~~~~~
 
+API token permissions are calculated based on ACLs containing their ID
+independent of those of their corresponding user. The resulting permission set
+on a given path is then intersected with that of the corresponding user.
+
+In practice this means:
+
+#. API tokens require their own ACL entries
+#. API tokens can never do more than their corresponding user
+
+Effective permissions
+~~~~~~~~~~~~~~~~~~~~~
+
+To calculate and display the effective permission set of a user or API token
+you can use the ``proxmox-backup-manager user permission`` command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager user permissions john@pbs -- path /datastore/store1
+  Privileges with (*) have the propagate flag set
+  
+  Path: /datastore/store1
+  - Datastore.Audit (*)
+  - Datastore.Backup (*)
+  - Datastore.Modify (*)
+  - Datastore.Prune (*)
+  - Datastore.Read (*)
+  
+  # proxmox-backup-manager acl update /datastore/store1 DatastoreBackup --auth-id 'john@pbs!client1'
+  # proxmox-backup-manager user permissions 'john@pbs!test' -- path /datastore/store1
+  Privileges with (*) have the propagate flag set
+  
+  Path: /datastore/store1
+  - Datastore.Backup (*)

etc/Makefile

@@ -1,9 +1,11 @@
 include ../defines.mk
 
-UNITS :=
+UNITS := \
+	proxmox-backup-daily-update.timer \
 
 DYNAMIC_UNITS := \
 	proxmox-backup-banner.service \
+	proxmox-backup-daily-update.service \
 	proxmox-backup.service \
 	proxmox-backup-proxy.service
 

etc/proxmox-backup-daily-update.service.in

@@ -0,0 +1,8 @@
+[Unit]
+Description=Daily Proxmox Backup Server update and maintenance activities
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=%LIBEXECDIR%/proxmox-backup/proxmox-daily-update

etc/proxmox-backup-daily-update.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily Proxmox Backup Server update and maintenance activities
+
+[Timer]
+OnCalendar=*-*-* 1:00
+RandomizedDelaySec=5h
+Persistent=true
+
+[Install]
+WantedBy=timers.target

etc/proxmox-backup-proxy.service.in

@@ -9,6 +9,7 @@ After=proxmox-backup.service
 Type=notify
 ExecStart=%LIBEXECDIR%/proxmox-backup/proxmox-backup-proxy
 ExecReload=/bin/kill -HUP $MAINPID
+PIDFile=/run/proxmox-backup/proxy.pid
 Restart=on-failure
 User=%PROXY_USER%
 Group=%PROXY_USER%

etc/proxmox-backup.service.in

@@ -7,6 +7,7 @@ After=network.target
 Type=notify
 ExecStart=%LIBEXECDIR%/proxmox-backup/proxmox-backup-api
 ExecReload=/bin/kill -HUP $MAINPID
+PIDFile=/run/proxmox-backup/api.pid
 Restart=on-failure
 
 [Install]

src/api2/access.rs

@@ -12,7 +12,6 @@ use proxmox::{http_err, list_subdirs_api_method};
 use crate::tools::ticket::{self, Empty, Ticket};
 use crate::auth_helpers::*;
 use crate::api2::types::*;
-use crate::tools::{FileLogOptions, FileLogger};
 
 use crate::config::acl as acl_config;
 use crate::config::acl::{PRIVILEGES, PRIV_SYS_AUDIT, PRIV_PERMISSIONS_MODIFY};
@@ -144,20 +143,13 @@ fn create_ticket(
     port: Option<u16>,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
-    let logger_options = FileLogOptions {
-        append: true,
-        prefix_time: true,
-        ..Default::default()
-    };
-    let mut auth_log = FileLogger::new("/var/log/proxmox-backup/api/auth.log", logger_options)?;
-
     match authenticate_user(&username, &password, path, privs, port) {
         Ok(true) => {
             let ticket = Ticket::new("PBS", &username)?.sign(private_auth_key(), None)?;
 
             let token = assemble_csrf_prevention_token(csrf_secret(), &username);
 
-            auth_log.log(format!("successful auth for user '{}'", username));
+            crate::server::rest::auth_logger()?.log(format!("successful auth for user '{}'", username));
 
             Ok(json!({
                 "username": username,
@@ -180,7 +172,7 @@ fn create_ticket(
                 username,
                 err.to_string()
             );
-            auth_log.log(&msg);
+            crate::server::rest::auth_logger()?.log(&msg);
             log::error!("{}", msg);
 
             Err(http_err!(UNAUTHORIZED, "permission check failed."))
@@ -244,7 +236,7 @@ fn change_password(
 #[api(
     input: {
         properties: {
-            auth_id: {
+            "auth-id": {
                 type: Authid,
                 optional: true,
             },

src/api2/access/acl.rs

@@ -169,7 +169,7 @@ pub fn read_acl(
                 optional: true,
                 schema: ACL_PROPAGATE_SCHEMA,
             },
-            auth_id: {
+            "auth-id": {
                 optional: true,
                 type: Authid,
             },

src/api2/access/user.rs

@@ -75,7 +75,7 @@ pub struct UserWithTokens {
     pub lastname: Option<String>,
     #[serde(skip_serializing_if="Option::is_none")]
     pub email: Option<String>,
-    #[serde(skip_serializing_if="Vec::is_empty")]
+    #[serde(skip_serializing_if="Vec::is_empty", default)]
     pub tokens: Vec<user::ApiToken>,
 }
 

src/api2/admin/datastore.rs

@@ -29,7 +29,7 @@ use crate::backup::*;
 use crate::config::datastore;
 use crate::config::cached_user_info::CachedUserInfo;
 
-use crate::server::WorkerTask;
+use crate::server::{jobstate::Job, WorkerTask};
 use crate::tools::{
     self,
     zip::{ZipEncoder, ZipEntry},
@@ -42,6 +42,7 @@ use crate::config::acl::{
     PRIV_DATASTORE_READ,
     PRIV_DATASTORE_PRUNE,
     PRIV_DATASTORE_BACKUP,
+    PRIV_DATASTORE_VERIFY,
 };
 
 fn check_priv_or_backup_owner(
@@ -537,7 +538,7 @@ pub fn status(
         schema: UPID_SCHEMA,
     },
     access: {
-        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ | PRIV_DATASTORE_BACKUP, true), // fixme
+        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_VERIFY | PRIV_DATASTORE_BACKUP, true),
     },
 )]
 /// Verify backups.
@@ -553,6 +554,7 @@ pub fn verify(
 ) -> Result<Value, Error> {
     let datastore = DataStore::lookup_datastore(&store)?;
 
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let worker_id;
 
     let mut backup_dir = None;
@@ -563,12 +565,18 @@ pub fn verify(
         (Some(backup_type), Some(backup_id), Some(backup_time)) => {
             worker_id = format!("{}:{}/{}/{:08X}", store, backup_type, backup_id, backup_time);
             let dir = BackupDir::new(backup_type, backup_id, backup_time)?;
+
+            check_priv_or_backup_owner(&datastore, dir.group(), &auth_id, PRIV_DATASTORE_VERIFY)?;
+
             backup_dir = Some(dir);
             worker_type = "verify_snapshot";
         }
         (Some(backup_type), Some(backup_id), None) => {
             worker_id = format!("{}:{}/{}", store, backup_type, backup_id);
             let group = BackupGroup::new(backup_type, backup_id);
+
+            check_priv_or_backup_owner(&datastore, &group, &auth_id, PRIV_DATASTORE_VERIFY)?;
+
             backup_group = Some(group);
             worker_type = "verify_group";
         }
@@ -578,13 +586,12 @@ pub fn verify(
         _ => bail!("parameters do not specify a backup group or snapshot"),
     }
 
-    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
 
     let upid_str = WorkerTask::new_thread(
         worker_type,
         Some(worker_id.clone()),
-        auth_id,
+        auth_id.clone(),
         to_stdout,
         move |worker| {
             let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*16)));
@@ -617,7 +624,16 @@ pub fn verify(
                 )?;
                 failed_dirs
             } else {
-                verify_all_backups(datastore, worker.clone(), worker.upid(), None)?
+                let privs = CachedUserInfo::new()?
+                    .lookup_privs(&auth_id, &["datastore", &store]);
+
+                let owner = if privs & PRIV_DATASTORE_VERIFY == 0 {
+                    Some(auth_id)
+                } else {
+                    None
+                };
+
+                verify_all_backups(datastore, worker.clone(), worker.upid(), owner, None)?
             };
             if failed_dirs.len() > 0 {
                 worker.log("Failed to verify following snapshots:");
@@ -840,20 +856,13 @@ fn start_garbage_collection(
     let datastore = DataStore::lookup_datastore(&store)?;
     let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
-    println!("Starting garbage collection on store {}", store);
+    let job =  Job::new("garbage_collection", &store)
+        .map_err(|_| format_err!("garbage collection already running"))?;
 
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
 
-    let upid_str = WorkerTask::new_thread(
+    let upid_str = crate::server::do_garbage_collection_job(job, datastore, &auth_id, None, to_stdout)
-        "garbage_collection",
+        .map_err(|err| format_err!("unable to start garbage collection job on datastore {} - {}", store, err))?;
-        Some(store.clone()),
-        auth_id.clone(),
-        to_stdout,
-        move |worker| {
-            worker.log(format!("starting garbage collection on store {}", store));
-            datastore.garbage_collection(&*worker, worker.upid())
-        },
-    )?;
 
     Ok(json!(upid_str))
 }
@@ -893,15 +902,7 @@ pub fn garbage_collection_status(
         type: Array,
         items: {
             description: "Datastore name and description.",
-            properties: {
+            type: DataStoreListItem,
-                store: {
-                    schema: DATASTORE_SCHEMA,
-                },
-                comment: {
-                    optional: true,
-                    schema: SINGLE_LINE_COMMENT_SCHEMA,
-                },
-            },
         },
     },
     access: {
@@ -913,7 +914,7 @@ fn get_datastore_list(
     _param: Value,
     _info: &ApiMethod,
     rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
+) -> Result<Vec<DataStoreListItem>, Error> {
 
     let (config, _digest) = datastore::config()?;
 
@@ -926,11 +927,12 @@ fn get_datastore_list(
         let user_privs = user_info.lookup_privs(&auth_id, &["datastore", &store]);
         let allowed = (user_privs & (PRIV_DATASTORE_AUDIT| PRIV_DATASTORE_BACKUP)) != 0;
         if allowed {
-            let mut entry = json!({ "store": store });
+            list.push(
-            if let Some(comment) = data["comment"].as_str() {
+                DataStoreListItem {
-                entry["comment"] = comment.into();
+                    store: store.clone(),
-            }
+                    comment: data["comment"].as_str().map(String::from),
-            list.push(entry);
+                }
+            );
         }
     }
 
@@ -1562,7 +1564,7 @@ fn get_rrd_stats(
         },
     },
     access: {
-        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ | PRIV_DATASTORE_BACKUP, true),
+        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_BACKUP, true),
     },
 )]
 /// Get "notes" for a specific backup
@@ -1578,7 +1580,7 @@ fn get_notes(
     let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
-    check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_READ)?;
+    check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_AUDIT)?;
 
     let (manifest, _) = datastore.load_manifest(&backup_dir)?;
 
@@ -1610,7 +1612,9 @@ fn get_notes(
         },
     },
     access: {
-        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY, true),
+        permission: &Permission::Privilege(&["datastore", "{store}"],
+                                           PRIV_DATASTORE_MODIFY | PRIV_DATASTORE_BACKUP,
+                                           true),
     },
 )]
 /// Set "notes" for a specific backup
@@ -1627,7 +1631,7 @@ fn set_notes(
     let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
-    check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_READ)?;
+    check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_MODIFY)?;
 
     datastore.update_manifest(&backup_dir,|manifest| {
         manifest.unprotected["notes"] = notes.into();

src/api2/admin/sync.rs

@@ -1,12 +1,15 @@
-use anyhow::{format_err, Error};
+use anyhow::{bail, format_err, Error};
 use serde_json::Value;
 
-use proxmox::api::{api, ApiMethod, Router, RpcEnvironment};
+use proxmox::api::{api, ApiMethod, Permission, Router, RpcEnvironment};
 use proxmox::api::router::SubdirMap;
 use proxmox::{list_subdirs_api_method, sortable};
 
 use crate::api2::types::*;
 use crate::api2::pull::do_sync_job;
+use crate::api2::config::sync::{check_sync_job_modify_access, check_sync_job_read_access};
+
+use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::sync::{self, SyncJobStatus, SyncJobConfig};
 use crate::server::UPID;
 use crate::server::jobstate::{Job, JobState};
@@ -27,6 +30,10 @@ use crate::tools::systemd::time::{
         type: Array,
         items: { type: sync::SyncJobStatus },
     },
+    access: {
+        description: "Limited to sync jobs where user has Datastore.Audit on target datastore, and Remote.Audit on source remote.",
+        permission: &Permission::Anybody,
+    },
 )]
 /// List all sync jobs
 pub fn list_sync_jobs(
@@ -35,6 +42,9 @@ pub fn list_sync_jobs(
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<SyncJobStatus>, Error> {
 
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
     let (config, digest) = sync::config()?;
 
     let mut list: Vec<SyncJobStatus> = config
@@ -46,6 +56,10 @@ pub fn list_sync_jobs(
             } else {
                 true
             }
+        })
+        .filter(|job: &SyncJobStatus| {
+            let as_config: SyncJobConfig = job.clone().into();
+            check_sync_job_read_access(&user_info, &auth_id, &as_config)
         }).collect();
 
     for job in &mut list {
@@ -89,7 +103,11 @@ pub fn list_sync_jobs(
                 schema: JOB_ID_SCHEMA,
             }
         }
-    }
+    },
+    access: {
+        description: "User needs Datastore.Backup on target datastore, and Remote.Read on source remote. Additionally, remove_vanished requires Datastore.Prune, and any owner other than the user themselves requires Datastore.Modify",
+        permission: &Permission::Anybody,
+    },
 )]
 /// Runs the sync jobs manually.
 fn run_sync_job(
@@ -97,11 +115,15 @@ fn run_sync_job(
     _info: &ApiMethod,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<String, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let (config, _digest) = sync::config()?;
     let sync_job: SyncJobConfig = config.lookup("sync", &id)?;
 
-    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    if !check_sync_job_modify_access(&user_info, &auth_id, &sync_job) {
+        bail!("permission check failed");
+    }
 
     let job = Job::new("syncjob", &id)?;
 

src/api2/admin/verify.rs

@@ -2,11 +2,16 @@ use anyhow::{format_err, Error};
 
 use proxmox::api::router::SubdirMap;
 use proxmox::{list_subdirs_api_method, sortable};
-use proxmox::api::{api, ApiMethod, Router, RpcEnvironment};
+use proxmox::api::{api, ApiMethod, Permission, Router, RpcEnvironment};
 
 use crate::api2::types::*;
 use crate::server::do_verification_job;
 use crate::server::jobstate::{Job, JobState};
+use crate::config::acl::{
+    PRIV_DATASTORE_AUDIT,
+    PRIV_DATASTORE_VERIFY,
+};
+use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::verify;
 use crate::config::verify::{VerificationJobConfig, VerificationJobStatus};
 use serde_json::Value;
@@ -23,10 +28,14 @@ use crate::server::UPID;
         },
     },
     returns: {
-        description: "List configured jobs and their status.",
+        description: "List configured jobs and their status (filtered by access)",
         type: Array,
         items: { type: verify::VerificationJobStatus },
     },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Audit or Datastore.Verify on datastore.",
+    },
 )]
 /// List all verification jobs
 pub fn list_verification_jobs(
@@ -34,6 +43,10 @@ pub fn list_verification_jobs(
     _param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<VerificationJobStatus>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_VERIFY;
 
     let (config, digest) = verify::config()?;
 
@@ -41,6 +54,11 @@ pub fn list_verification_jobs(
         .convert_to_typed_array("verification")?
         .into_iter()
         .filter(|job: &VerificationJobStatus| {
+            let privs = user_info.lookup_privs(&auth_id, &["datastore", &job.store]);
+            if privs & required_privs == 0 {
+                return false;
+            }
+
             if let Some(store) = &store {
                 &job.store == store
             } else {
@@ -90,7 +108,11 @@ pub fn list_verification_jobs(
                 schema: JOB_ID_SCHEMA,
             }
         }
-    }
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Verify on job's datastore.",
+    },
 )]
 /// Runs a verification job manually.
 fn run_verification_job(
@@ -98,10 +120,13 @@ fn run_verification_job(
     _info: &ApiMethod,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<String, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
     let (config, _digest) = verify::config()?;
     let verification_job: VerificationJobConfig = config.lookup("verification", &id)?;
 
-    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    user_info.check_privs(&auth_id, &["datastore", &verification_job.store], PRIV_DATASTORE_VERIFY, true)?;
 
     let job = Job::new("verificationjob", &id)?;
 

src/api2/config/datastore.rs

@@ -5,6 +5,7 @@ use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, Router, RpcEnvironment, Permission};
+use proxmox::api::schema::parse_property_string;
 use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
@@ -68,6 +69,14 @@ pub fn list_datastores(
                 optional: true,
                 schema: SINGLE_LINE_COMMENT_SCHEMA,
             },
+            "notify-user": {
+                optional: true,
+                type: Userid,
+            },
+            "notify": {
+                optional: true,
+                schema: DATASTORE_NOTIFY_STRING_SCHEMA,
+            },
             "gc-schedule": {
                 optional: true,
                 schema: GC_SCHEDULE_SCHEMA,
@@ -187,6 +196,12 @@ pub enum DeletableProperty {
     keep_monthly,
     /// Delete the keep-yearly property
     keep_yearly,
+    /// Delete the verify-new property
+    verify_new,
+    /// Delete the notify-user property
+    notify_user,
+    /// Delete the notify property
+    notify,
 }
 
 #[api(
@@ -200,6 +215,14 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: SINGLE_LINE_COMMENT_SCHEMA,
             },
+            "notify-user": {
+                optional: true,
+                type: Userid,
+            },
+            "notify": {
+                optional: true,
+                schema: DATASTORE_NOTIFY_STRING_SCHEMA,
+            },
             "gc-schedule": {
                 optional: true,
                 schema: GC_SCHEDULE_SCHEMA,
@@ -232,6 +255,12 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_YEARLY,
             },
+            "verify-new": {
+                description: "If enabled, all new backups will be verified right after completion.",
+                type: bool,
+                optional: true,
+                default: false,
+            },
             delete: {
                 description: "List of properties to delete.",
                 type: Array,
@@ -262,6 +291,9 @@ pub fn update_datastore(
     keep_weekly: Option<u64>,
     keep_monthly: Option<u64>,
     keep_yearly: Option<u64>,
+    verify_new: Option<bool>,
+    notify: Option<String>,
+    notify_user: Option<Userid>,
     delete: Option<Vec<DeletableProperty>>,
     digest: Option<String>,
 ) -> Result<(), Error> {
@@ -290,6 +322,9 @@ pub fn update_datastore(
                 DeletableProperty::keep_weekly => { data.keep_weekly = None; },
                 DeletableProperty::keep_monthly => { data.keep_monthly = None; },
                 DeletableProperty::keep_yearly => { data.keep_yearly = None; },
+                DeletableProperty::verify_new => { data.verify_new = None; },
+                DeletableProperty::notify => { data.notify = None; },
+                DeletableProperty::notify_user => { data.notify_user = None; },
             }
         }
     }
@@ -322,6 +357,19 @@ pub fn update_datastore(
     if keep_monthly.is_some() { data.keep_monthly = keep_monthly; }
     if keep_yearly.is_some() { data.keep_yearly = keep_yearly; }
 
+    if let Some(notify_str) = notify {
+        let value = parse_property_string(&notify_str, &DatastoreNotify::API_SCHEMA)?;
+        let notify: DatastoreNotify = serde_json::from_value(value)?;
+        if let  DatastoreNotify { gc: None, verify: None, sync: None } = notify {
+            data.notify = None;
+        } else {
+            data.notify = Some(notify_str);
+        }
+    }
+    if verify_new.is_some() { data.verify_new = verify_new; }
+
+    if notify_user.is_some() { data.notify_user = notify_user; }
+
     config.set_data(&name, "datastore", &data)?;
 
     datastore::save_config(&config)?;

src/api2/config/remote.rs

@@ -1,11 +1,14 @@
-use anyhow::{bail, Error};
+use anyhow::{bail, format_err, Error};
 use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
+use proxmox::http_err;
 use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
+use crate::client::{HttpClient, HttpClientOptions};
+use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::remote;
 use crate::config::acl::{PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY};
 
@@ -22,7 +25,8 @@ use crate::config::acl::{PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY};
         },
     },
     access: {
-        permission: &Permission::Privilege(&["remote"], PRIV_REMOTE_AUDIT, false),
+        description: "List configured remotes filtered by Remote.Audit privileges",
+        permission: &Permission::Anybody,
     },
 )]
 /// List all remotes
@@ -31,16 +35,25 @@ pub fn list_remotes(
     _info: &ApiMethod,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<remote::Remote>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let (config, digest) = remote::config()?;
 
     let mut list: Vec<remote::Remote> = config.convert_to_typed_array("remote")?;
-
     // don't return password in api
     for remote in &mut list {
         remote.password = "".to_string();
     }
 
+    let list = list
+        .into_iter()
+        .filter(|remote| {
+            let privs = user_info.lookup_privs(&auth_id, &["remote", &remote.name]);
+            privs & PRIV_REMOTE_AUDIT != 0
+        })
+        .collect();
+
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
     Ok(list)
 }
@@ -290,10 +303,83 @@ pub fn delete_remote(name: String, digest: Option<String>) -> Result<(), Error>
     Ok(())
 }
 
+/// Helper to get client for remote.cfg entry
+pub async fn remote_client(remote: remote::Remote) -> Result<HttpClient, Error> {
+    let options = HttpClientOptions::new()
+        .password(Some(remote.password.clone()))
+        .fingerprint(remote.fingerprint.clone());
+
+    let client = HttpClient::new(
+        &remote.host,
+        remote.port.unwrap_or(8007),
+        &remote.userid,
+        options)?;
+    let _auth_info = client.login() // make sure we can auth
+        .await
+        .map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?;
+
+    Ok(client)
+}
+
+
+#[api(
+    input: {
+        properties: {
+            name: {
+                schema: REMOTE_ID_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["remote", "{name}"], PRIV_REMOTE_AUDIT, false),
+    },
+    returns: {
+        description: "List the accessible datastores.",
+        type: Array,
+        items: {
+            description: "Datastore name and description.",
+            type: DataStoreListItem,
+        },
+    },
+)]
+/// List datastores of a remote.cfg entry
+pub async fn scan_remote_datastores(name: String) -> Result<Vec<DataStoreListItem>, Error> {
+    let (remote_config, _digest) = remote::config()?;
+    let remote: remote::Remote = remote_config.lookup("remote", &name)?;
+
+    let map_remote_err = |api_err| {
+        http_err!(INTERNAL_SERVER_ERROR,
+                  "failed to scan remote '{}' - {}",
+                  &name,
+                  api_err)
+    };
+
+    let client = remote_client(remote)
+        .await
+        .map_err(map_remote_err)?;
+    let api_res = client
+        .get("api2/json/admin/datastore", None)
+        .await
+        .map_err(map_remote_err)?;
+    let parse_res = match api_res.get("data") {
+        Some(data) => serde_json::from_value::<Vec<DataStoreListItem>>(data.to_owned()),
+        None => bail!("remote {} did not return any datastore list data", &name),
+    };
+
+    match parse_res {
+        Ok(parsed) => Ok(parsed),
+        Err(_) => bail!("Failed to parse remote scan api result."),
+    }
+}
+
+const SCAN_ROUTER: Router = Router::new()
+    .get(&API_METHOD_SCAN_REMOTE_DATASTORES);
+
 const ITEM_ROUTER: Router = Router::new()
     .get(&API_METHOD_READ_REMOTE)
     .put(&API_METHOD_UPDATE_REMOTE)
-    .delete(&API_METHOD_DELETE_REMOTE);
+    .delete(&API_METHOD_DELETE_REMOTE)
+    .subdirs(&[("scan", &SCAN_ROUTER)]);
 
 pub const ROUTER: Router = Router::new()
     .get(&API_METHOD_LIST_REMOTES)

src/api2/config/sync.rs

@@ -2,13 +2,73 @@ use anyhow::{bail, Error};
 use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
 
-use proxmox::api::{api, Router, RpcEnvironment};
+use proxmox::api::{api, Permission, Router, RpcEnvironment};
 use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
+
+use crate::config::acl::{
+    PRIV_DATASTORE_AUDIT,
+    PRIV_DATASTORE_BACKUP,
+    PRIV_DATASTORE_MODIFY,
+    PRIV_DATASTORE_PRUNE,
+    PRIV_REMOTE_AUDIT,
+    PRIV_REMOTE_READ,
+};
+
+use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::sync::{self, SyncJobConfig};
 
-// fixme: add access permissions
+pub fn check_sync_job_read_access(
+    user_info: &CachedUserInfo,
+    auth_id: &Authid,
+    job: &SyncJobConfig,
+) -> bool {
+    let datastore_privs = user_info.lookup_privs(&auth_id, &["datastore", &job.store]);
+    if datastore_privs & PRIV_DATASTORE_AUDIT == 0 {
+        return false;
+    }
+
+    let remote_privs = user_info.lookup_privs(&auth_id, &["remote", &job.remote]);
+    remote_privs & PRIV_REMOTE_AUDIT != 0
+}
+
+// user can run the corresponding pull job
+pub fn check_sync_job_modify_access(
+    user_info: &CachedUserInfo,
+    auth_id: &Authid,
+    job: &SyncJobConfig,
+) -> bool {
+    let datastore_privs = user_info.lookup_privs(&auth_id, &["datastore", &job.store]);
+    if datastore_privs & PRIV_DATASTORE_BACKUP == 0 {
+        return false;
+    }
+
+    if let Some(true) = job.remove_vanished {
+        if datastore_privs & PRIV_DATASTORE_PRUNE == 0 {
+            return false;
+        }
+    }
+
+    let correct_owner = match job.owner {
+        Some(ref owner) => {
+            owner == auth_id
+                || (owner.is_token()
+                    && !auth_id.is_token()
+                    && owner.user() == auth_id.user())
+        },
+        // default sync owner
+        None => auth_id == Authid::root_auth_id(),
+    };
+
+    // same permission as changing ownership after syncing
+    if !correct_owner && datastore_privs & PRIV_DATASTORE_MODIFY == 0 {
+        return false;
+    }
+
+    let remote_privs = user_info.lookup_privs(&auth_id, &["remote", &job.remote, &job.remote_store]);
+    remote_privs & PRIV_REMOTE_READ != 0
+}
 
 #[api(
     input: {
@@ -19,12 +79,18 @@ use crate::config::sync::{self, SyncJobConfig};
         type: Array,
         items: { type: sync::SyncJobConfig },
     },
+    access: {
+        description: "Limited to sync job entries where user has Datastore.Audit on target datastore, and Remote.Audit on source remote.",
+        permission: &Permission::Anybody,
+    },
 )]
 /// List all sync jobs
 pub fn list_sync_jobs(
     _param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<SyncJobConfig>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let (config, digest) = sync::config()?;
 
@@ -32,7 +98,11 @@ pub fn list_sync_jobs(
 
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
-    Ok(list)
+    let list = list
+        .into_iter()
+        .filter(|sync_job| check_sync_job_read_access(&user_info, &auth_id, &sync_job))
+        .collect();
+   Ok(list)
 }
 
 #[api(
@@ -45,6 +115,10 @@ pub fn list_sync_jobs(
             store: {
                 schema: DATASTORE_SCHEMA,
             },
+            owner: {
+                type: Authid,
+                optional: true,
+            },
             remote: {
                 schema: REMOTE_ID_SCHEMA,
             },
@@ -65,13 +139,25 @@ pub fn list_sync_jobs(
             },
         },
     },
+    access: {
+        description: "User needs Datastore.Backup on target datastore, and Remote.Read on source remote. Additionally, remove_vanished requires Datastore.Prune, and any owner other than the user themselves requires Datastore.Modify",
+        permission: &Permission::Anybody,
+    },
 )]
 /// Create a new sync job.
-pub fn create_sync_job(param: Value) -> Result<(), Error> {
+pub fn create_sync_job(
+    param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let sync_job: sync::SyncJobConfig = serde_json::from_value(param.clone())?;
+    if !check_sync_job_modify_access(&user_info, &auth_id, &sync_job) {
+        bail!("permission check failed");
+    }
 
     let (mut config, _digest) = sync::config()?;
 
@@ -100,15 +186,26 @@ pub fn create_sync_job(param: Value) -> Result<(), Error> {
         description: "The sync job configuration.",
         type: sync::SyncJobConfig,
     },
+    access: {
+        description: "Limited to sync job entries where user has Datastore.Audit on target datastore, and Remote.Audit on source remote.",
+        permission: &Permission::Anybody,
+    },
 )]
 /// Read a sync job configuration.
 pub fn read_sync_job(
     id: String,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<SyncJobConfig, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
     let (config, digest) = sync::config()?;
 
     let sync_job = config.lookup("sync", &id)?;
+    if !check_sync_job_read_access(&user_info, &auth_id, &sync_job) {
+        bail!("permission check failed");
+    }
+
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
     Ok(sync_job)
@@ -120,6 +217,8 @@ pub fn read_sync_job(
 #[allow(non_camel_case_types)]
 /// Deletable property name
 pub enum DeletableProperty {
+    /// Delete the owner property.
+    owner,
     /// Delete the comment property.
     comment,
     /// Delete the job schedule.
@@ -139,6 +238,10 @@ pub enum DeletableProperty {
                 schema: DATASTORE_SCHEMA,
                 optional: true,
             },
+            owner: {
+                type: Authid,
+                optional: true,
+            },
             remote: {
                 schema: REMOTE_ID_SCHEMA,
                 optional: true,
@@ -173,11 +276,16 @@ pub enum DeletableProperty {
             },
         },
     },
+    access: {
+        permission: &Permission::Anybody,
+        description: "User needs Datastore.Backup on target datastore, and Remote.Read on source remote. Additionally, remove_vanished requires Datastore.Prune, and any owner other than the user themselves requires Datastore.Modify",
+    },
 )]
 /// Update sync job config.
 pub fn update_sync_job(
     id: String,
     store: Option<String>,
+    owner: Option<Authid>,
     remote: Option<String>,
     remote_store: Option<String>,
     remove_vanished: Option<bool>,
@@ -185,7 +293,10 @@ pub fn update_sync_job(
     schedule: Option<String>,
     delete: Option<Vec<DeletableProperty>>,
     digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
@@ -202,6 +313,7 @@ pub fn update_sync_job(
      if let Some(delete) = delete {
         for delete_prop in delete {
             match delete_prop {
+                DeletableProperty::owner => { data.owner = None; },
                 DeletableProperty::comment => { data.comment = None; },
                 DeletableProperty::schedule => { data.schedule = None; },
                 DeletableProperty::remove_vanished => { data.remove_vanished = None; },
@@ -221,11 +333,15 @@ pub fn update_sync_job(
     if let Some(store) = store { data.store = store; }
     if let Some(remote) = remote { data.remote = remote; }
     if let Some(remote_store) = remote_store { data.remote_store = remote_store; }
-        
+    if let Some(owner) = owner { data.owner = Some(owner); }
-    
+
     if schedule.is_some() { data.schedule = schedule; }
     if remove_vanished.is_some() { data.remove_vanished = remove_vanished; }
 
+    if !check_sync_job_modify_access(&user_info, &auth_id, &data) {
+        bail!("permission check failed");
+    }
+
     config.set_data(&id, "sync", &data)?;
 
     sync::save_config(&config)?;
@@ -246,9 +362,19 @@ pub fn update_sync_job(
             },
         },
     },
+    access: {
+        permission: &Permission::Anybody,
+        description: "User needs Datastore.Backup on target datastore, and Remote.Read on source remote. Additionally, remove_vanished requires Datastore.Prune, and any owner other than the user themselves requires Datastore.Modify",
+    },
 )]
 /// Remove a sync job configuration
-pub fn delete_sync_job(id: String, digest: Option<String>) -> Result<(), Error> {
+pub fn delete_sync_job(
+    id: String,
+    digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
@@ -259,10 +385,15 @@ pub fn delete_sync_job(id: String, digest: Option<String>) -> Result<(), Error>
         crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
     }
 
-    match config.sections.get(&id) {
+    match config.lookup("sync", &id) {
-        Some(_) => { config.sections.remove(&id); },
+        Ok(job) => {
-        None => bail!("job '{}' does not exist.", id),
+            if !check_sync_job_modify_access(&user_info, &auth_id, &job) {
-    }
+                bail!("permission check failed");
+            }
+            config.sections.remove(&id);
+        },
+        Err(_) => { bail!("job '{}' does not exist.", id) },
+    };
 
     sync::save_config(&config)?;
 
@@ -280,3 +411,116 @@ pub const ROUTER: Router = Router::new()
     .get(&API_METHOD_LIST_SYNC_JOBS)
     .post(&API_METHOD_CREATE_SYNC_JOB)
     .match_all("id", &ITEM_ROUTER);
+
+
+#[test]
+fn sync_job_access_test() -> Result<(), Error> {
+    let (user_cfg, _) = crate::config::user::test_cfg_from_str(r###"
+user: noperm@pbs
+
+user: read@pbs
+
+user: write@pbs
+
+"###).expect("test user.cfg is not parsable");
+    let acl_tree = crate::config::acl::AclTree::from_raw(r###"
+acl:1:/datastore/localstore1:read@pbs,write@pbs:DatastoreAudit
+acl:1:/datastore/localstore1:write@pbs:DatastoreBackup
+acl:1:/datastore/localstore2:write@pbs:DatastorePowerUser
+acl:1:/datastore/localstore3:write@pbs:DatastoreAdmin
+acl:1:/remote/remote1:read@pbs,write@pbs:RemoteAudit
+acl:1:/remote/remote1/remotestore1:write@pbs:RemoteSyncOperator
+"###).expect("test acl.cfg is not parsable");
+
+    let user_info = CachedUserInfo::test_new(user_cfg, acl_tree);
+
+    let root_auth_id = Authid::root_auth_id();
+
+    let no_perm_auth_id: Authid = "noperm@pbs".parse()?;
+    let read_auth_id: Authid = "read@pbs".parse()?;
+    let write_auth_id: Authid = "write@pbs".parse()?;
+
+    let mut job = SyncJobConfig {
+        id: "regular".to_string(),
+        remote: "remote0".to_string(),
+        remote_store: "remotestore1".to_string(),
+        store: "localstore0".to_string(),
+        owner: Some(write_auth_id.clone()),
+        comment: None,
+        remove_vanished: None,
+        schedule: None,
+    };
+
+    // should work without ACLs
+    assert_eq!(check_sync_job_read_access(&user_info, &root_auth_id, &job), true);
+    assert_eq!(check_sync_job_modify_access(&user_info, &root_auth_id, &job), true);
+
+    // user without permissions must fail
+    assert_eq!(check_sync_job_read_access(&user_info, &no_perm_auth_id, &job), false);
+    assert_eq!(check_sync_job_modify_access(&user_info, &no_perm_auth_id, &job), false);
+
+    // reading without proper read permissions on either remote or local must fail
+    assert_eq!(check_sync_job_read_access(&user_info, &read_auth_id, &job), false);
+
+    // reading without proper read permissions on local end must fail
+    job.remote = "remote1".to_string();
+    assert_eq!(check_sync_job_read_access(&user_info, &read_auth_id, &job), false);
+
+    // reading without proper read permissions on remote end must fail
+    job.remote = "remote0".to_string();
+    job.store = "localstore1".to_string();
+    assert_eq!(check_sync_job_read_access(&user_info, &read_auth_id, &job), false);
+
+    // writing without proper write permissions on either end must fail
+    job.store = "localstore0".to_string();
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
+
+    // writing without proper write permissions on local end must fail
+    job.remote = "remote1".to_string();
+
+    // writing without proper write permissions on remote end must fail
+    job.remote = "remote0".to_string();
+    job.store = "localstore1".to_string();
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
+
+    // reset remote to one where users have access
+    job.remote = "remote1".to_string();
+
+    // user with read permission can only read, but not modify/run
+    assert_eq!(check_sync_job_read_access(&user_info, &read_auth_id, &job), true);
+    job.owner = Some(read_auth_id.clone());
+    assert_eq!(check_sync_job_modify_access(&user_info, &read_auth_id, &job), false);
+    job.owner = None;
+    assert_eq!(check_sync_job_modify_access(&user_info, &read_auth_id, &job), false);
+    job.owner = Some(write_auth_id.clone());
+    assert_eq!(check_sync_job_modify_access(&user_info, &read_auth_id, &job), false);
+
+    // user with simple write permission can modify/run
+    assert_eq!(check_sync_job_read_access(&user_info, &write_auth_id, &job), true);
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), true);
+
+    // but can't modify/run with deletion
+    job.remove_vanished = Some(true);
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
+
+    // unless they have Datastore.Prune as well
+    job.store = "localstore2".to_string();
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), true);
+
+    // changing owner is not possible
+    job.owner = Some(read_auth_id.clone());
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
+
+    // also not to the default 'root@pam'
+    job.owner = None;
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
+
+    // unless they have Datastore.Modify as well
+    job.store = "localstore3".to_string();
+    job.owner = Some(read_auth_id.clone());
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), true);
+    job.owner = None;
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), true);
+
+    Ok(())
+}

src/api2/config/verify.rs

@@ -2,10 +2,18 @@ use anyhow::{bail, Error};
 use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
 
-use proxmox::api::{api, Router, RpcEnvironment};
+use proxmox::api::{api, Permission, Router, RpcEnvironment};
 use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
+
+use crate::config::acl::{
+    PRIV_DATASTORE_AUDIT,
+    PRIV_DATASTORE_VERIFY,
+};
+
+use crate::config::cached_user_info::CachedUserInfo;
+
 use crate::config::verify::{self, VerificationJobConfig};
 
 #[api(
@@ -17,17 +25,32 @@ use crate::config::verify::{self, VerificationJobConfig};
         type: Array,
         items: { type: verify::VerificationJobConfig },
     },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Audit or Datastore.Verify on datastore.",
+    },
 )]
 /// List all verification jobs
 pub fn list_verification_jobs(
     _param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<VerificationJobConfig>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_VERIFY;
 
     let (config, digest) = verify::config()?;
 
     let list = config.convert_to_typed_array("verification")?;
 
+    let list = list.into_iter()
+        .filter(|job: &VerificationJobConfig| {
+            let privs = user_info.lookup_privs(&auth_id, &["datastore", &job.store]);
+
+            privs & required_privs != 00
+        }).collect();
+
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
     Ok(list)
@@ -61,15 +84,26 @@ pub fn list_verification_jobs(
                 schema: VERIFICATION_SCHEDULE_SCHEMA,
             },
         }
-    }
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Verify on job's datastore.",
+    },
 )]
 /// Create a new verification job.
-pub fn create_verification_job(param: Value) -> Result<(), Error> {
+pub fn create_verification_job(
-
+    param: Value,
-    let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+    rpcenv: &mut dyn RpcEnvironment
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let verification_job: verify::VerificationJobConfig = serde_json::from_value(param.clone())?;
 
+    user_info.check_privs(&auth_id, &["datastore", &verification_job.store], PRIV_DATASTORE_VERIFY, false)?;
+
+    let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
     let (mut config, _digest) = verify::config()?;
 
     if let Some(_) = config.sections.get(&verification_job.id) {
@@ -97,15 +131,26 @@ pub fn create_verification_job(param: Value) -> Result<(), Error> {
         description: "The verification job configuration.",
         type: verify::VerificationJobConfig,
     },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Audit or Datastore.Verify on job's datastore.",
+    },
 )]
 /// Read a verification job configuration.
 pub fn read_verification_job(
     id: String,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<VerificationJobConfig, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
     let (config, digest) = verify::config()?;
 
-    let verification_job = config.lookup("verification", &id)?;
+    let verification_job: verify::VerificationJobConfig = config.lookup("verification", &id)?;
+
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_VERIFY;
+    user_info.check_privs(&auth_id, &["datastore", &verification_job.store], required_privs, true)?;
+
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
     Ok(verification_job)
@@ -167,6 +212,10 @@ pub enum DeletableProperty {
             },
         },
     },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Verify on job's datastore.",
+    },
 )]
 /// Update verification job config.
 pub fn update_verification_job(
@@ -178,7 +227,10 @@ pub fn update_verification_job(
     schedule: Option<String>,
     delete: Option<Vec<DeletableProperty>>,
     digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
@@ -192,7 +244,10 @@ pub fn update_verification_job(
 
     let mut data: verify::VerificationJobConfig = config.lookup("verification", &id)?;
 
-     if let Some(delete) = delete {
+    // check existing store
+    user_info.check_privs(&auth_id, &["datastore", &data.store], PRIV_DATASTORE_VERIFY, true)?;
+
+    if let Some(delete) = delete {
         for delete_prop in delete {
             match delete_prop {
                 DeletableProperty::IgnoreVerified => { data.ignore_verified = None; },
@@ -212,7 +267,12 @@ pub fn update_verification_job(
         }
     }
 
-    if let Some(store) = store { data.store = store; }
+    if let Some(store) = store {
+        // check new store
+        user_info.check_privs(&auth_id, &["datastore", &store], PRIV_DATASTORE_VERIFY, true)?;
+        data.store = store;
+    }
+
 
     if ignore_verified.is_some() { data.ignore_verified = ignore_verified; }
     if outdated_after.is_some() { data.outdated_after = outdated_after; }
@@ -238,14 +298,27 @@ pub fn update_verification_job(
             },
         },
     },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Verify on job's datastore.",
+    },
 )]
 /// Remove a verification job configuration
-pub fn delete_verification_job(id: String, digest: Option<String>) -> Result<(), Error> {
+pub fn delete_verification_job(
+    id: String,
+    digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let (mut config, expected_digest) = verify::config()?;
 
+    let job: verify::VerificationJobConfig = config.lookup("verification", &id)?;
+    user_info.check_privs(&auth_id, &["datastore", &job.store], PRIV_DATASTORE_VERIFY, true)?;
+
     if let Some(ref digest) = digest {
         let digest = proxmox::tools::hex_to_digest(digest)?;
         crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;

src/api2/node.rs

@@ -24,20 +24,21 @@ use crate::server::WorkerTask;
 use crate::tools;
 use crate::tools::ticket::{self, Empty, Ticket};
 
+pub mod apt;
 pub mod disks;
 pub mod dns;
 pub mod network;
 pub mod tasks;
+pub mod subscription;
 
 pub(crate) mod rrd;
 
-mod apt;
 mod journal;
 mod services;
 mod status;
-mod subscription;
 mod syslog;
 mod time;
+mod report;
 
 pub const SHELL_CMD_SCHEMA: Schema = StringSchema::new("The command to run.")
     .format(&ApiStringFormat::Enum(&[
@@ -310,6 +311,7 @@ pub const SUBDIRS: SubdirMap = &[
     ("dns", &dns::ROUTER),
     ("journal", &journal::ROUTER),
     ("network", &network::ROUTER),
+    ("report", &report::ROUTER),
     ("rrd", &rrd::ROUTER),
     ("services", &services::ROUTER),
     ("status", &status::ROUTER),

src/api2/node/apt.rs

@@ -1,302 +1,16 @@
-use std::collections::HashSet;
-
-use apt_pkg_native::Cache;
 use anyhow::{Error, bail, format_err};
 use serde_json::{json, Value};
 
-use proxmox::{list_subdirs_api_method, const_regex};
+use proxmox::list_subdirs_api_method;
 use proxmox::api::{api, RpcEnvironment, RpcEnvironmentType, Permission};
 use proxmox::api::router::{Router, SubdirMap};
 
 use crate::server::WorkerTask;
-use crate::tools::http;
+use crate::tools::{apt, http};
 
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
 use crate::api2::types::{Authid, APTUpdateInfo, NODE_SCHEMA, UPID_SCHEMA};
 
-const_regex! {
-    VERSION_EPOCH_REGEX = r"^\d+:";
-    FILENAME_EXTRACT_REGEX = r"^.*/.*?_(.*)_Packages$";
-}
-
-// FIXME: once the 'changelog' API call switches over to 'apt-get changelog' only,
-// consider removing this function entirely, as it's value is never used anywhere
-// then (widget-toolkit doesn't use the value either)
-fn get_changelog_url(
-    package: &str,
-    filename: &str,
-    version: &str,
-    origin: &str,
-    component: &str,
-) -> Result<String, Error> {
-    if origin == "" {
-        bail!("no origin available for package {}", package);
-    }
-
-    if origin == "Debian" {
-        let mut command = std::process::Command::new("apt-get");
-        command.arg("changelog");
-        command.arg("--print-uris");
-        command.arg(package);
-        let output = crate::tools::run_command(command, None)?; // format: 'http://foo/bar' package.changelog
-        let output = match output.splitn(2, ' ').next() {
-            Some(output) => {
-                if output.len() < 2 {
-                    bail!("invalid output (URI part too short) from 'apt-get changelog --print-uris': {}", output)
-                }
-                output[1..output.len()-1].to_owned()
-            },
-            None => bail!("invalid output from 'apt-get changelog --print-uris': {}", output)
-        };
-        return Ok(output);
-    } else if origin == "Proxmox" {
-        // FIXME: Use above call to 'apt changelog <pkg> --print-uris' as well.
-        // Currently not possible as our packages do not have a URI set in their Release file.
-        let version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(version, "");
-
-        let base = match (FILENAME_EXTRACT_REGEX.regex_obj)().captures(filename) {
-            Some(captures) => {
-                let base_capture = captures.get(1);
-                match base_capture {
-                    Some(base_underscore) => base_underscore.as_str().replace("_", "/"),
-                    None => bail!("incompatible filename, cannot find regex group")
-                }
-            },
-            None => bail!("incompatible filename, doesn't match regex")
-        };
-
-        return Ok(format!("http://download.proxmox.com/{}/{}_{}.changelog",
-                          base, package, version));
-    }
-
-    bail!("unknown origin ({}) or component ({})", origin, component)
-}
-
-struct FilterData<'a> {
-    // this is version info returned by APT
-    installed_version: Option<&'a str>,
-    candidate_version: &'a str,
-
-    // this is the version info the filter is supposed to check
-    active_version: &'a str,
-}
-
-enum PackagePreSelect {
-    OnlyInstalled,
-    OnlyNew,
-    All,
-}
-
-fn list_installed_apt_packages<F: Fn(FilterData) -> bool>(
-    filter: F,
-    only_versions_for: Option<&str>,
-) -> Vec<APTUpdateInfo> {
-
-    let mut ret = Vec::new();
-    let mut depends = HashSet::new();
-
-    // note: this is not an 'apt update', it just re-reads the cache from disk
-    let mut cache = Cache::get_singleton();
-    cache.reload();
-
-    let mut cache_iter = match only_versions_for {
-        Some(name) => cache.find_by_name(name),
-        None => cache.iter()
-    };
-
-    loop {
-
-        match cache_iter.next() {
-            Some(view) => {
-                let di = if only_versions_for.is_some() {
-                    query_detailed_info(
-                        PackagePreSelect::All,
-                        &filter,
-                        view,
-                        None
-                    )
-                } else {
-                    query_detailed_info(
-                        PackagePreSelect::OnlyInstalled,
-                        &filter,
-                        view,
-                        Some(&mut depends)
-                    )
-                };
-                if let Some(info) = di {
-                    ret.push(info);
-                }
-
-                if only_versions_for.is_some() {
-                    break;
-                }
-            },
-            None => {
-                drop(cache_iter);
-                // also loop through missing dependencies, as they would be installed
-                for pkg in depends.iter() {
-                    let mut iter = cache.find_by_name(&pkg);
-                    let view = match iter.next() {
-                        Some(view) => view,
-                        None => continue // package not found, ignore
-                    };
-
-                    let di = query_detailed_info(
-                        PackagePreSelect::OnlyNew,
-                        &filter,
-                        view,
-                        None
-                    );
-                    if let Some(info) = di {
-                        ret.push(info);
-                    }
-                }
-                break;
-            }
-        }
-    }
-
-    return ret;
-}
-
-fn query_detailed_info<'a, F, V>(
-    pre_select: PackagePreSelect,
-    filter: F,
-    view: V,
-    depends: Option<&mut HashSet<String>>,
-) -> Option<APTUpdateInfo>
-where
-    F: Fn(FilterData) -> bool,
-    V: std::ops::Deref<Target = apt_pkg_native::sane::PkgView<'a>>
-{
-    let current_version = view.current_version();
-    let candidate_version = view.candidate_version();
-
-    let (current_version, candidate_version) = match pre_select {
-        PackagePreSelect::OnlyInstalled => match (current_version, candidate_version) {
-            (Some(cur), Some(can)) => (Some(cur), can), // package installed and there is an update
-            (Some(cur), None) => (Some(cur.clone()), cur), // package installed and up-to-date
-            (None, Some(_)) => return None, // package could be installed
-            (None, None) => return None, // broken
-        },
-        PackagePreSelect::OnlyNew => match (current_version, candidate_version) {
-            (Some(_), Some(_)) => return None,
-            (Some(_), None) => return None,
-            (None, Some(can)) => (None, can),
-            (None, None) => return None,
-        },
-        PackagePreSelect::All => match (current_version, candidate_version) {
-            (Some(cur), Some(can)) => (Some(cur), can),
-            (Some(cur), None) => (Some(cur.clone()), cur),
-            (None, Some(can)) => (None, can),
-            (None, None) => return None,
-        },
-    };
-
-    // get additional information via nested APT 'iterators'
-    let mut view_iter = view.versions();
-    while let Some(ver) = view_iter.next() {
-
-        let package = view.name();
-        let version = ver.version();
-        let mut origin_res = "unknown".to_owned();
-        let mut section_res = "unknown".to_owned();
-        let mut priority_res = "unknown".to_owned();
-        let mut change_log_url = "".to_owned();
-        let mut short_desc = package.clone();
-        let mut long_desc = "".to_owned();
-
-        let fd = FilterData {
-            installed_version: current_version.as_deref(),
-            candidate_version: &candidate_version,
-            active_version: &version,
-        };
-
-        if filter(fd) {
-            if let Some(section) = ver.section() {
-                section_res = section;
-            }
-
-            if let Some(prio) = ver.priority_type() {
-                priority_res = prio;
-            }
-
-            // assume every package has only one origin file (not
-            // origin, but origin *file*, for some reason those seem to
-            // be different concepts in APT)
-            let mut origin_iter = ver.origin_iter();
-            let origin = origin_iter.next();
-            if let Some(origin) = origin {
-
-                if let Some(sd) = origin.short_desc() {
-                    short_desc = sd;
-                }
-
-                if let Some(ld) = origin.long_desc() {
-                    long_desc = ld;
-                }
-
-                // the package files appear in priority order, meaning
-                // the one for the candidate version is first - this is fine
-                // however, as the source package should be the same for all
-                // versions anyway
-                let mut pkg_iter = origin.file();
-                let pkg_file = pkg_iter.next();
-                if let Some(pkg_file) = pkg_file {
-                    if let Some(origin_name) = pkg_file.origin() {
-                        origin_res = origin_name;
-                    }
-
-                    let filename = pkg_file.file_name();
-                    let component = pkg_file.component();
-
-                    // build changelog URL from gathered information
-                    // ignore errors, use empty changelog instead
-                    let url = get_changelog_url(&package, &filename,
-                        &version, &origin_res, &component);
-                    if let Ok(url) = url {
-                        change_log_url = url;
-                    }
-                }
-            }
-
-            if let Some(depends) = depends {
-                let mut dep_iter = ver.dep_iter();
-                loop {
-                    let dep = match dep_iter.next() {
-                        Some(dep) if dep.dep_type() != "Depends" => continue,
-                        Some(dep) => dep,
-                        None => break
-                    };
-
-                    let dep_pkg = dep.target_pkg();
-                    let name = dep_pkg.name();
-
-                    depends.insert(name);
-                }
-            }
-
-            return Some(APTUpdateInfo {
-                package,
-                title: short_desc,
-                arch: view.arch(),
-                description: long_desc,
-                change_log_url,
-                origin: origin_res,
-                version: candidate_version.clone(),
-                old_version: match current_version {
-                    Some(vers) => vers,
-                    None => "".to_owned()
-                },
-                priority: priority_res,
-                section: section_res,
-            });
-        }
-    }
-
-    return None;
-}
-
 #[api(
     input: {
         properties: {
@@ -308,19 +22,60 @@ where
     returns: {
         description: "A list of packages with available updates.",
         type: Array,
-        items: { type: APTUpdateInfo },
+        items: {
+            type: APTUpdateInfo
+        },
     },
+    protected: true,
     access: {
         permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
     },
 )]
 /// List available APT updates
 fn apt_update_available(_param: Value) -> Result<Value, Error> {
-    let all_upgradeable = list_installed_apt_packages(|data| {
+
-        data.candidate_version == data.active_version &&
+    match apt::pkg_cache_expired() {
-        data.installed_version != Some(data.candidate_version)
+        Ok(false) => {
-    }, None);
+            if let Ok(Some(cache)) = apt::read_pkg_state() {
-    Ok(json!(all_upgradeable))
+                return Ok(json!(cache.package_status));
+            }
+        },
+        _ => (),
+    }
+
+    let cache = apt::update_cache()?;
+
+    return Ok(json!(cache.package_status));
+}
+
+fn do_apt_update(worker: &WorkerTask, quiet: bool) -> Result<(), Error> {
+    if !quiet { worker.log("starting apt-get update") }
+
+    // TODO: set proxy /etc/apt/apt.conf.d/76pbsproxy like PVE
+
+    let mut command = std::process::Command::new("apt-get");
+    command.arg("update");
+
+    // apt "errors" quite easily, and run_command is a bit rigid, so handle this inline for now.
+    let output = command.output()
+        .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
+
+    if !quiet {
+        worker.log(String::from_utf8(output.stdout)?);
+    }
+
+    // TODO: improve run_command to allow outputting both, stderr and stdout
+    if !output.status.success() {
+        if output.status.code().is_some() {
+            let msg = String::from_utf8(output.stderr)
+                .map(|m| if m.is_empty() { String::from("no error message") } else { m })
+                .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
+            worker.warn(msg);
+        } else {
+            bail!("terminated by signal");
+        }
+    }
+    Ok(())
 }
 
 #[api(
@@ -330,6 +85,13 @@ fn apt_update_available(_param: Value) -> Result<Value, Error> {
             node: {
                 schema: NODE_SCHEMA,
             },
+            notify: {
+                type: bool,
+                description: r#"Send notification mail about new package updates availanle to the
+                    email address configured for 'root@pam')."#,
+                optional: true,
+                default: false,
+            },
             quiet: {
                 description: "Only produces output suitable for logging, omitting progress indicators.",
                 type: bool,
@@ -347,26 +109,46 @@ fn apt_update_available(_param: Value) -> Result<Value, Error> {
 )]
 /// Update the APT database
 pub fn apt_update_database(
+    notify: Option<bool>,
     quiet: Option<bool>,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<String, Error> {
 
     let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
+    // FIXME: change to non-option in signature and drop below once we have proxmox-api-macro 0.2.3
     let quiet = quiet.unwrap_or(API_METHOD_APT_UPDATE_DATABASE_PARAM_DEFAULT_QUIET);
+    let notify = notify.unwrap_or(API_METHOD_APT_UPDATE_DATABASE_PARAM_DEFAULT_NOTIFY);
 
     let upid_str = WorkerTask::new_thread("aptupdate", None, auth_id, to_stdout, move |worker| {
-        if !quiet { worker.log("starting apt-get update") }
+        do_apt_update(&worker, quiet)?;
 
-        // TODO: set proxy /etc/apt/apt.conf.d/76pbsproxy like PVE
+        let mut cache = apt::update_cache()?;
 
-        let mut command = std::process::Command::new("apt-get");
+        if notify {
-        command.arg("update");
+            let mut notified = match cache.notified {
+                Some(notified) => notified,
+                None => std::collections::HashMap::new(),
+            };
+            let mut to_notify: Vec<&APTUpdateInfo> = Vec::new();
 
-        let output = crate::tools::run_command(command, None)?;
+            for pkg in &cache.package_status {
-        if !quiet { worker.log(output) }
+                match notified.insert(pkg.package.to_owned(), pkg.version.to_owned()) {
-
+                    Some(notified_version) => {
-        // TODO: add mail notify for new updates like PVE
+                        if notified_version != pkg.version {
+                            to_notify.push(pkg);
+                        }
+                    },
+                    None => to_notify.push(pkg),
+                }
+            }
+            if !to_notify.is_empty() {
+                to_notify.sort_unstable_by_key(|k| &k.package);
+                crate::server::send_updates_available(&to_notify)?;
+            }
+            cache.notified = Some(notified);
+            apt::write_pkg_cache(&cache)?;
+        }
 
         Ok(())
     })?;
@@ -406,7 +188,7 @@ fn apt_get_changelog(
     let name = crate::tools::required_string_param(&param, "name")?.to_owned();
     let version = param["version"].as_str();
 
-    let pkg_info = list_installed_apt_packages(|data| {
+    let pkg_info = apt::list_installed_apt_packages(|data| {
         match version {
             Some(version) => version == data.active_version,
             None => data.active_version == data.candidate_version

src/api2/node/report.rs

@@ -0,0 +1,35 @@
+use anyhow::Error;
+use proxmox::api::{api, ApiMethod, Permission, Router, RpcEnvironment};
+use serde_json::{json, Value};
+
+use crate::api2::types::*;
+use crate::config::acl::PRIV_SYS_AUDIT;
+use crate::server::generate_report;
+
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+        },
+    },
+    returns: {
+        type: String,
+        description: "Returns report of the node"
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "status"], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// Generate a report
+fn get_report(
+    _param: Value,
+    _info: &ApiMethod,
+    _rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Value, Error> {
+    Ok(json!(generate_report()))
+}
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_GET_REPORT);

src/api2/node/subscription.rs

@@ -7,7 +7,7 @@ use crate::tools;
 use crate::tools::subscription::{self, SubscriptionStatus, SubscriptionInfo};
 use crate::config::acl::{PRIV_SYS_AUDIT,PRIV_SYS_MODIFY};
 use crate::config::cached_user_info::CachedUserInfo;
-use crate::api2::types::{NODE_SCHEMA, Authid};
+use crate::api2::types::{NODE_SCHEMA, SUBSCRIPTION_KEY_SCHEMA, Authid};
 
 #[api(
     input: {
@@ -29,7 +29,7 @@ use crate::api2::types::{NODE_SCHEMA, Authid};
     },
 )]
 /// Check and update subscription status.
-fn check_subscription(
+pub fn check_subscription(
     force: bool,
 ) -> Result<(), Error> {
     // FIXME: drop once proxmox-api-macro is bumped to >> 5.0.0-1
@@ -82,7 +82,7 @@ fn check_subscription(
     },
 )]
 /// Read subscription info.
-fn get_subscription(
+pub fn get_subscription(
     _param: Value,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<SubscriptionInfo, Error> {
@@ -124,9 +124,7 @@ fn get_subscription(
                 schema: NODE_SCHEMA,
             },
             key: {
-                description: "Proxmox Backup Server subscription key",
+                schema: SUBSCRIPTION_KEY_SCHEMA,
-                type: String,
-                max_length: 32,
             },
         },
     },
@@ -136,7 +134,7 @@ fn get_subscription(
     },
 )]
 /// Set a subscription key and check it.
-fn set_subscription(
+pub fn set_subscription(
     key: String,
 ) -> Result<(), Error> {
 
@@ -164,7 +162,7 @@ fn set_subscription(
     },
 )]
 /// Delete subscription info.
-fn delete_subscription() -> Result<(), Error> {
+pub fn delete_subscription() -> Result<(), Error> {
 
     subscription::delete_subscription()
         .map_err(|err| format_err!("Deleting subscription failed: {}", err))?;

src/api2/node/tasks.rs

@@ -1,7 +1,7 @@
 use std::fs::File;
 use std::io::{BufRead, BufReader};
 
-use anyhow::{Error};
+use anyhow::{bail, Error};
 use serde_json::{json, Value};
 
 use proxmox::api::{api, Router, RpcEnvironment, Permission};
@@ -9,19 +9,87 @@ use proxmox::api::router::SubdirMap;
 use proxmox::{identity, list_subdirs_api_method, sortable};
 
 use crate::tools;
+
 use crate::api2::types::*;
+use crate::api2::pull::check_pull_privs;
+
 use crate::server::{self, UPID, TaskState, TaskListInfoIterator};
-use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
+use crate::config::acl::{
+    PRIV_DATASTORE_MODIFY,
+    PRIV_DATASTORE_VERIFY,
+    PRIV_SYS_AUDIT,
+    PRIV_SYS_MODIFY,
+};
 use crate::config::cached_user_info::CachedUserInfo;
 
+// matches respective job execution privileges
+fn check_job_privs(auth_id: &Authid, user_info: &CachedUserInfo, upid: &UPID) -> Result<(), Error> {
+    match (upid.worker_type.as_str(), &upid.worker_id) {
+        ("verificationjob", Some(workerid)) => {
+            if let Some(captures) = VERIFICATION_JOB_WORKER_ID_REGEX.captures(&workerid) {
+                if let Some(store) = captures.get(1) {
+                    return user_info.check_privs(&auth_id,
+                                                 &["datastore", store.as_str()],
+                                                 PRIV_DATASTORE_VERIFY,
+                                                 true);
+                }
+            }
+        },
+        ("syncjob", Some(workerid)) => {
+            if let Some(captures) = SYNC_JOB_WORKER_ID_REGEX.captures(&workerid) {
+                let remote = captures.get(1);
+                let remote_store = captures.get(2);
+                let local_store = captures.get(3);
+
+                if let (Some(remote), Some(remote_store), Some(local_store)) =
+                    (remote, remote_store, local_store) {
+
+                    return check_pull_privs(&auth_id,
+                                            local_store.as_str(),
+                                            remote.as_str(),
+                                            remote_store.as_str(),
+                                            false);
+                }
+            }
+        },
+        ("garbage_collection", Some(workerid)) => {
+            return user_info.check_privs(&auth_id,
+                                         &["datastore", &workerid],
+                                         PRIV_DATASTORE_MODIFY,
+                                         true)
+        },
+        ("prune", Some(workerid)) => {
+            return user_info.check_privs(&auth_id,
+                                         &["datastore",
+                                         &workerid],
+                                         PRIV_DATASTORE_MODIFY,
+                                         true);
+        },
+        _ => bail!("not a scheduled job task"),
+    };
+
+    bail!("not a scheduled job task");
+}
+
 fn check_task_access(auth_id: &Authid, upid: &UPID) -> Result<(), Error> {
     let task_auth_id = &upid.auth_id;
     if auth_id == task_auth_id
         || (task_auth_id.is_token() && &Authid::from(task_auth_id.user().clone()) == auth_id) {
+        // task owner can always read
         Ok(())
     } else {
         let user_info = CachedUserInfo::new()?;
-        user_info.check_privs(auth_id, &["system", "tasks"], PRIV_SYS_AUDIT, false)
+
+        let task_privs = user_info.lookup_privs(auth_id, &["system", "tasks"]);
+        if task_privs & PRIV_SYS_AUDIT != 0 {
+            // allowed to read all tasks in general
+            Ok(())
+        } else if check_job_privs(&auth_id, &user_info, upid).is_ok() {
+            // job which the user/token could have configured/manually executed
+            Ok(())
+        } else {
+            bail!("task access not allowed");
+        }
     }
 }
 
@@ -271,7 +339,7 @@ fn stop_task(
             },
             limit: {
                 type: u64,
-                description: "Only list this amount of tasks.",
+                description: "Only list this amount of tasks. (0 means no limit)",
                 default: 50,
                 optional: true,
             },
@@ -296,6 +364,29 @@ fn stop_task(
                 type: String,
                 description: "Only list tasks from this user.",
             },
+            since: {
+                type: i64,
+                description: "Only list tasks since this UNIX epoch.",
+                optional: true,
+            },
+            until: {
+                type: i64,
+                description: "Only list tasks until this UNIX epoch.",
+                optional: true,
+            },
+            typefilter: {
+                optional: true,
+                type: String,
+                description: "Only list tasks whose type contains this.",
+            },
+            statusfilter: {
+                optional: true,
+                type: Array,
+                description: "Only list tasks which have any one of the listed status.",
+                items: {
+                    type: TaskStateType,
+                },
+            },
         },
     },
     returns: {
@@ -315,6 +406,10 @@ pub fn list_tasks(
     errors: bool,
     running: bool,
     userfilter: Option<String>,
+    since: Option<i64>,
+    until: Option<i64>,
+    typefilter: Option<String>,
+    statusfilter: Option<Vec<TaskStateType>>,
     param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<TaskListItem>, Error> {
@@ -328,9 +423,23 @@ pub fn list_tasks(
     let store = param["store"].as_str();
 
     let list = TaskListInfoIterator::new(running)?;
+    let limit = if limit > 0 { limit as usize } else { usize::MAX };
 
     let result: Vec<TaskListItem> = list
-        .take_while(|info| !info.is_err())
+        .skip_while(|info| {
+            match (info, until) {
+                (Ok(info), Some(until)) => info.upid.starttime > until,
+                (Ok(_), None) => false,
+                (Err(_), _) => false,
+            }
+        })
+        .take_while(|info| {
+            match (info, since) {
+                (Ok(info), Some(since)) => info.upid.starttime > since,
+                (Ok(_), None) => true,
+                (Err(_), _) => false,
+            }
+        })
         .filter_map(|info| {
         let info = match info {
             Ok(info) => info,
@@ -364,19 +473,31 @@ pub fn list_tasks(
             }
         }
 
-        match info.state {
+        if let Some(typefilter) = &typefilter {
-            Some(_) if running => return None,
+            if !info.upid.worker_type.contains(typefilter) {
-            Some(crate::server::TaskState::OK { .. }) if errors => return None,
+                return None;
+            }
+        }
+
+        match (&info.state, &statusfilter) {
+            (Some(_), _) if running => return None,
+            (Some(crate::server::TaskState::OK { .. }), _) if errors => return None,
+            (Some(state), Some(filters)) => {
+                if !filters.contains(&state.tasktype()) {
+                    return None;
+                }
+            },
+            (None, Some(_)) => return None,
             _ => {},
         }
 
         Some(info.into())
     }).skip(start as usize)
-        .take(limit as usize)
+        .take(limit)
         .collect();
 
     let mut count = result.len() + start as usize;
-    if result.len() > 0 && result.len() >= limit as usize { // we have a 'virtual' entry as long as we have any new
+    if result.len() > 0 && result.len() >= limit { // we have a 'virtual' entry as long as we have any new
         count += 1;
     }
 

src/api2/pull.rs

@@ -9,7 +9,7 @@ use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
 
 use crate::server::{WorkerTask, jobstate::Job};
 use crate::backup::DataStore;
-use crate::client::{HttpClient, HttpClientOptions, BackupRepository, pull::pull_store};
+use crate::client::{HttpClient, BackupRepository, pull::pull_store};
 use crate::api2::types::*;
 use crate::config::{
     remote,
@@ -50,17 +50,9 @@ pub async fn get_pull_parameters(
     let (remote_config, _digest) = remote::config()?;
     let remote: remote::Remote = remote_config.lookup("remote", remote)?;
 
-    let options = HttpClientOptions::new()
-        .password(Some(remote.password.clone()))
-        .fingerprint(remote.fingerprint.clone());
-
     let src_repo = BackupRepository::new(Some(remote.userid.clone()), Some(remote.host.clone()), remote.port, remote_store.to_string());
 
-    let client = HttpClient::new(&src_repo.host(), src_repo.port(), &src_repo.auth_id(), options)?;
+    let client = crate::api2::config::remote::remote_client(remote).await?;
-    let _auth_info = client.login() // make sure we can auth
-        .await
-        .map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?;
-
 
     Ok((client, src_repo, tgt_store))
 }
@@ -72,14 +64,18 @@ pub fn do_sync_job(
     schedule: Option<String>,
 ) -> Result<String, Error> {
 
-    let job_id = job.jobname().to_string();
+    let job_id = format!("{}:{}:{}:{}",
+                         sync_job.remote,
+                         sync_job.remote_store,
+                         sync_job.store,
+                         job.jobname());
     let worker_type = job.jobtype().to_string();
 
-    let email = crate::server::lookup_user_email(auth_id.user());
+    let (email, notify) = crate::server::lookup_datastore_notify_settings(&sync_job.store);
 
     let upid_str = WorkerTask::spawn(
         &worker_type,
-        Some(job.jobname().to_string()),
+        Some(job_id.clone()),
         auth_id.clone(),
         false,
         move |worker| async move {
@@ -92,6 +88,7 @@ pub fn do_sync_job(
             let worker_future = async move {
 
                 let delete = sync_job.remove_vanished.unwrap_or(true);
+                let sync_owner = sync_job.owner.unwrap_or(Authid::root_auth_id().clone());
                 let (client, src_repo, tgt_store) = get_pull_parameters(&sync_job.store, &sync_job.remote, &sync_job.remote_store).await?;
 
                 worker.log(format!("Starting datastore sync job '{}'", job_id));
@@ -101,9 +98,7 @@ pub fn do_sync_job(
                 worker.log(format!("Sync datastore '{}' from '{}/{}'",
                         sync_job.store, sync_job.remote, sync_job.remote_store));
 
-                let backup_auth_id = Authid::backup_auth_id();
+                crate::client::pull::pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, sync_owner).await?;
-
-                crate::client::pull::pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, backup_auth_id.clone()).await?;
 
                 worker.log(format!("sync job '{}' end", &job_id));
 
@@ -127,7 +122,7 @@ pub fn do_sync_job(
             }
 
             if let Some(email) = email {
-                if let Err(err) = crate::server::send_sync_status(&email, &sync_job2, &result) {
+                if let Err(err) = crate::server::send_sync_status(&email, notify, &sync_job2, &result) {
                     eprintln!("send sync notification failed: {}", err);
                 }
             }

src/api2/status.rs

@@ -17,17 +17,13 @@ use crate::api2::types::{
     RRDMode,
     RRDTimeFrameResolution,
     Authid,
-    TaskListItem,
-    TaskStateType,
 };
 
-use crate::server;
 use crate::backup::{DataStore};
 use crate::config::datastore;
 use crate::tools::statistics::{linear_regression};
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::acl::{
-    PRIV_SYS_AUDIT,
     PRIV_DATASTORE_AUDIT,
     PRIV_DATASTORE_BACKUP,
 };
@@ -179,103 +175,8 @@ fn datastore_status(
     Ok(list.into())
 }
 
-#[api(
-    input: {
-        properties: {
-            since: {
-                type: i64,
-                description: "Only list tasks since this UNIX epoch.",
-                optional: true,
-            },
-            typefilter: {
-                optional: true,
-                type: String,
-                description: "Only list tasks, whose type contains this string.",
-            },
-            statusfilter: {
-                optional: true,
-                type: Array,
-                description: "Only list tasks which have any one of the listed status.",
-                items: {
-                    type: TaskStateType,
-                },
-            },
-        },
-    },
-    returns: {
-        description: "A list of tasks.",
-        type: Array,
-        items: { type: TaskListItem },
-    },
-    access: {
-        description: "Users can only see there own tasks, unless the have Sys.Audit on /system/tasks.",
-        permission: &Permission::Anybody,
-    },
-)]
-/// List tasks.
-pub fn list_tasks(
-    since: Option<i64>,
-    typefilter: Option<String>,
-    statusfilter: Option<Vec<TaskStateType>>,
-    _param: Value,
-    rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Vec<TaskListItem>, Error> {
-
-    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
-    let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&auth_id, &["system", "tasks"]);
-
-    let list_all = (user_privs & PRIV_SYS_AUDIT) != 0;
-    let since = since.unwrap_or_else(|| 0);
-
-    let list: Vec<TaskListItem> = server::TaskListInfoIterator::new(false)?
-        .take_while(|info| {
-            match info {
-                Ok(info) => info.upid.starttime > since,
-                Err(_) => false
-            }
-        })
-        .filter_map(|info| {
-            match info {
-                Ok(info) => {
-                    if list_all || info.upid.auth_id == auth_id {
-                        if let Some(filter) = &typefilter {
-                            if !info.upid.worker_type.contains(filter) {
-                                return None;
-                            }
-                        }
-
-                        if let Some(filters) = &statusfilter {
-                            if let Some(state) = &info.state {
-                                let statetype = match state {
-                                    server::TaskState::OK { .. } => TaskStateType::OK,
-                                    server::TaskState::Unknown { .. } => TaskStateType::Unknown,
-                                    server::TaskState::Error { .. } => TaskStateType::Error,
-                                    server::TaskState::Warning { .. } => TaskStateType::Warning,
-                                };
-
-                                if !filters.contains(&statetype) {
-                                    return None;
-                                }
-                            }
-                        }
-
-                        Some(Ok(TaskListItem::from(info)))
-                    } else {
-                        None
-                    }
-                }
-                Err(err) => Some(Err(err))
-            }
-        })
-        .collect::<Result<Vec<TaskListItem>, Error>>()?;
-
-    Ok(list.into())
-}
-
 const SUBDIRS: SubdirMap = &[
     ("datastore-usage", &Router::new().get(&API_METHOD_DATASTORE_STATUS)),
-    ("tasks", &Router::new().get(&API_METHOD_LIST_TASKS)),
 ];
 
 pub const ROUTER: Router = Router::new()

src/api2/types/mod.rs

@@ -5,7 +5,7 @@ use proxmox::api::{api, schema::*};
 use proxmox::const_regex;
 use proxmox::{IPRE, IPRE_BRACKET, IPV4RE, IPV6RE, IPV4OCTET, IPV6H16, IPV6LS32};
 
-use crate::backup::CryptMode;
+use crate::backup::{CryptMode, BACKUP_ID_REGEX};
 use crate::server::UPID;
 
 #[macro_use]
@@ -59,6 +59,11 @@ const_regex!{
     /// any identifier command line tools work with.
     pub PROXMOX_SAFE_ID_REGEX = concat!(r"^", PROXMOX_SAFE_ID_REGEX_STR!(), r"$");
 
+    /// Regex for verification jobs 'DATASTORE:ACTUAL_JOB_ID'
+    pub VERIFICATION_JOB_WORKER_ID_REGEX = concat!(r"^(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):");
+    /// Regex for sync jobs 'REMOTE:REMOTE_DATASTORE:LOCAL_DATASTORE:ACTUAL_JOB_ID'
+    pub SYNC_JOB_WORKER_ID_REGEX = concat!(r"^(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):");
+
     pub SINGLE_LINE_COMMENT_REGEX = r"^[[:^cntrl:]]*$";
 
     pub HOSTNAME_REGEX = r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9])?)$";
@@ -73,6 +78,8 @@ const_regex!{
 
     pub ACL_PATH_REGEX = concat!(r"^(?:/|", r"(?:/", PROXMOX_SAFE_ID_REGEX_STR!(), ")+", r")$");
 
+    pub SUBSCRIPTION_KEY_REGEX = concat!(r"^pbs(?:[cbsp])-[0-9a-f]{10}$");
+
     pub BLOCKDEVICE_NAME_REGEX = r"^(:?(:?h|s|x?v)d[a-z]+)|(:?nvme\d+n\d+)$";
 
     pub ZPOOL_NAME_REGEX = r"^[a-zA-Z][a-z0-9A-Z\-_.:]+$";
@@ -99,6 +106,9 @@ pub const CERT_FINGERPRINT_SHA256_FORMAT: ApiStringFormat =
 pub const PROXMOX_SAFE_ID_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&PROXMOX_SAFE_ID_REGEX);
 
+pub const BACKUP_ID_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&BACKUP_ID_REGEX);
+
 pub const SINGLE_LINE_COMMENT_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&SINGLE_LINE_COMMENT_REGEX);
 
@@ -129,6 +139,9 @@ pub const CIDR_V6_FORMAT: ApiStringFormat =
 pub const CIDR_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&CIDR_REGEX);
 
+pub const SUBSCRIPTION_KEY_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&SUBSCRIPTION_KEY_REGEX);
+
 pub const BLOCKDEVICE_NAME_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&BLOCKDEVICE_NAME_REGEX);
 
@@ -271,7 +284,7 @@ pub const BACKUP_TYPE_SCHEMA: Schema =
 
 pub const BACKUP_ID_SCHEMA: Schema =
     StringSchema::new("Backup ID.")
-    .format(&PROXMOX_SAFE_ID_FORMAT)
+    .format(&BACKUP_ID_FORMAT)
     .schema();
 
 pub const BACKUP_TIME_SCHEMA: Schema =
@@ -348,6 +361,12 @@ pub const DNS_NAME_OR_IP_SCHEMA: Schema = StringSchema::new("DNS name or IP addr
     .format(&DNS_NAME_OR_IP_FORMAT)
     .schema();
 
+pub const SUBSCRIPTION_KEY_SCHEMA: Schema = StringSchema::new("Proxmox Backup Server subscription key.")
+    .format(&SUBSCRIPTION_KEY_FORMAT)
+    .min_length(15)
+    .max_length(16)
+    .schema();
+
 pub const BLOCKDEVICE_NAME_SCHEMA: Schema = StringSchema::new("Block device name (/sys/block/<name>).")
     .format(&BLOCKDEVICE_NAME_FORMAT)
     .min_length(3)
@@ -356,6 +375,25 @@ pub const BLOCKDEVICE_NAME_SCHEMA: Schema = StringSchema::new("Block device name
 
 // Complex type definitions
 
+#[api(
+    properties: {
+        store: {
+            schema: DATASTORE_SCHEMA,
+        },
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+    },
+)]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all="kebab-case")]
+/// Basic information about a datastore.
+pub struct DataStoreListItem {
+    pub store: String,
+    pub comment: Option<String>,
+}
+
 #[api(
     properties: {
         "backup-type": {
@@ -1115,7 +1153,7 @@ pub enum RRDTimeFrameResolution {
 }
 
 #[api()]
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "PascalCase")]
 /// Describes a package for which an update is available.
 pub struct APTUpdateInfo {
@@ -1140,3 +1178,48 @@ pub struct APTUpdateInfo {
     /// URL under which the package's changelog can be retrieved
     pub change_log_url: String,
 }
+
+#[api()]
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// When do we send notifications
+pub enum Notify {
+    /// Never send notification
+    Never,
+    /// Send notifications for failed and sucessful jobs
+    Always,
+    /// Send notifications for failed jobs only
+    Error,
+}
+
+#[api(
+    properties: {
+        gc: {
+            type: Notify,
+            optional: true,
+        },
+        verify: {
+            type: Notify,
+            optional: true,
+        },
+        sync: {
+            type: Notify,
+            optional: true,
+        },
+    },
+)]
+#[derive(Debug, Serialize, Deserialize)]
+/// Datastore notify settings
+pub struct DatastoreNotify {
+    /// Garbage collection settings
+    pub gc: Option<Notify>,
+    /// Verify job setting
+    pub verify: Option<Notify>,
+    /// Sync job setting
+    pub sync: Option<Notify>,
+}
+
+pub const DATASTORE_NOTIFY_STRING_SCHEMA: Schema = StringSchema::new(
+    "Datastore notification setting")
+    .format(&ApiStringFormat::PropertyString(&DatastoreNotify::API_SCHEMA))
+    .schema();

src/api2/types/userid.rs

@@ -450,11 +450,6 @@ impl Userid {
         &self.data
     }
 
-    /// Get the "backup@pam" user id.
-    pub fn backup_userid() -> &'static Self {
-        &*BACKUP_USERID
-    }
-
     /// Get the "root@pam" user id.
     pub fn root_userid() -> &'static Self {
         &*ROOT_USERID
@@ -462,7 +457,6 @@ impl Userid {
 }
 
 lazy_static! {
-    pub static ref BACKUP_USERID: Userid = Userid::new("backup@pam".to_string(), 6);
     pub static ref ROOT_USERID: Userid = Userid::new("root@pam".to_string(), 4);
 }
 
@@ -474,6 +468,12 @@ impl PartialEq for Userid {
     }
 }
 
+impl From<Authid> for Userid {
+    fn from(authid: Authid) -> Self {
+        authid.user
+    }
+}
+
 impl From<(Username, Realm)> for Userid {
     fn from(parts: (Username, Realm)) -> Self {
         Self::from((parts.0.as_ref(), parts.1.as_ref()))
@@ -590,11 +590,6 @@ impl Authid {
         }
     }
 
-    /// Get the "backup@pam" auth id.
-    pub fn backup_auth_id() -> &'static Self {
-        &*BACKUP_AUTHID
-    }
-
     /// Get the "root@pam" auth id.
     pub fn root_auth_id() -> &'static Self {
         &*ROOT_AUTHID
@@ -602,7 +597,6 @@ impl Authid {
 }
 
 lazy_static! {
-    pub static ref BACKUP_AUTHID: Authid = Authid::from(Userid::new("backup@pam".to_string(), 6));
     pub static ref ROOT_AUTHID: Authid = Authid::from(Userid::new("root@pam".to_string(), 4));
 }
 

src/backup/backup_info.rs

@@ -1,37 +1,31 @@
 use crate::tools;
 
 use anyhow::{bail, format_err, Error};
-use regex::Regex;
 use std::os::unix::io::RawFd;
 
 use std::path::{PathBuf, Path};
-use lazy_static::lazy_static;
+
+use proxmox::const_regex;
 
 use super::manifest::MANIFEST_BLOB_NAME;
 
-macro_rules! BACKUP_ID_RE { () => (r"[A-Za-z0-9][A-Za-z0-9_-]+") }
+macro_rules! BACKUP_ID_RE { () => (r"[A-Za-z0-9_][A-Za-z0-9._\-]*") }
 macro_rules! BACKUP_TYPE_RE { () => (r"(?:host|vm|ct)") }
 macro_rules! BACKUP_TIME_RE { () => (r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z") }
 
-lazy_static!{
+const_regex!{
-    static ref BACKUP_FILE_REGEX: Regex = Regex::new(
+    BACKUP_FILE_REGEX = r"^.*\.([fd]idx|blob)$";
-        r"^.*\.([fd]idx|blob)$").unwrap();
 
-    static ref BACKUP_TYPE_REGEX: Regex = Regex::new(
+    BACKUP_TYPE_REGEX = concat!(r"^(", BACKUP_TYPE_RE!(), r")$");
-        concat!(r"^(", BACKUP_TYPE_RE!(), r")$")).unwrap();
 
-    static ref BACKUP_ID_REGEX: Regex = Regex::new(
+    pub BACKUP_ID_REGEX = concat!(r"^", BACKUP_ID_RE!(), r"$");
-        concat!(r"^", BACKUP_ID_RE!(), r"$")).unwrap();
 
-    static ref BACKUP_DATE_REGEX: Regex = Regex::new(
+    BACKUP_DATE_REGEX = concat!(r"^", BACKUP_TIME_RE!() ,r"$");
-        concat!(r"^", BACKUP_TIME_RE!() ,r"$")).unwrap();
 
-    static ref GROUP_PATH_REGEX: Regex = Regex::new(
+    GROUP_PATH_REGEX = concat!(r"^(", BACKUP_TYPE_RE!(), ")/(", BACKUP_ID_RE!(), r")$");
-        concat!(r"^(", BACKUP_TYPE_RE!(), ")/(", BACKUP_ID_RE!(), r")$")).unwrap();
-
-    static ref SNAPSHOT_PATH_REGEX: Regex = Regex::new(
-        concat!(r"^(", BACKUP_TYPE_RE!(), ")/(", BACKUP_ID_RE!(), ")/(", BACKUP_TIME_RE!(), r")$")).unwrap();
 
+    SNAPSHOT_PATH_REGEX = concat!(
+        r"^(", BACKUP_TYPE_RE!(), ")/(", BACKUP_ID_RE!(), ")/(", BACKUP_TIME_RE!(), r")$");
 }
 
 /// BackupGroup is a directory containing a list of BackupDir

src/backup/datastore.rs

@@ -458,38 +458,35 @@ impl DataStore {
     ) -> Result<(), Error> {
 
         let image_list = self.list_images()?;
-
         let image_count = image_list.len();
 
         let mut done = 0;
-
         let mut last_percentage: usize = 0;
 
-        for path in image_list {
+        for img in image_list {
 
             worker.check_abort()?;
             tools::fail_on_shutdown()?;
 
-            let full_path = self.chunk_store.relative_path(&path);
+            let path = self.chunk_store.relative_path(&img);
-            match std::fs::File::open(&full_path) {
+            match std::fs::File::open(&path) {
                 Ok(file) => {
-                    if let Ok(archive_type) = archive_type(&path) {
+                    if let Ok(archive_type) = archive_type(&img) {
                         if archive_type == ArchiveType::FixedIndex {
-                            let index = FixedIndexReader::new(file)?;
+                            let index = FixedIndexReader::new(file).map_err(|e| {
-                            self.index_mark_used_chunks(index, &path, status, worker)?;
+                                format_err!("can't read index '{}' - {}", path.to_string_lossy(), e)
+                            })?;
+                            self.index_mark_used_chunks(index, &img, status, worker)?;
                         } else if archive_type == ArchiveType::DynamicIndex {
-                            let index = DynamicIndexReader::new(file)?;
+                            let index = DynamicIndexReader::new(file).map_err(|e| {
-                            self.index_mark_used_chunks(index, &path, status, worker)?;
+                                format_err!("can't read index '{}' - {}", path.to_string_lossy(), e)
+                            })?;
+                            self.index_mark_used_chunks(index, &img, status, worker)?;
                         }
                     }
                 }
-                Err(err) => {
+                Err(err) if err.kind() == io::ErrorKind::NotFound => (), // ignore vanished files
-                    if err.kind() == std::io::ErrorKind::NotFound {
+                Err(err) => bail!("can't open index {} - {}", path.to_string_lossy(), err),
-                        // simply ignore vanished files
-                    } else {
-                        return Err(err.into());
-                    }
-                }
             }
             done += 1;
 

src/backup/dynamic_index.rs

@@ -95,6 +95,18 @@ impl DynamicIndexReader {
 
         let header_size = std::mem::size_of::<DynamicIndexHeader>();
 
+        let rawfd = file.as_raw_fd();
+        let stat = match nix::sys::stat::fstat(rawfd) {
+            Ok(stat) => stat,
+            Err(err) => bail!("fstat failed - {}", err),
+        };
+
+        let size = stat.st_size as usize;
+
+        if size < header_size {
+            bail!("index too small ({})", stat.st_size);
+        }
+
         let header: Box<DynamicIndexHeader> = unsafe { file.read_host_value_boxed()? };
 
         if header.magic != super::DYNAMIC_SIZED_CHUNK_INDEX_1_0 {
@@ -103,13 +115,7 @@ impl DynamicIndexReader {
 
         let ctime = proxmox::tools::time::epoch_i64();
 
-        let rawfd = file.as_raw_fd();
+        let index_size = stat.st_size as usize - header_size;
-
-        let stat = nix::sys::stat::fstat(rawfd)?;
-
-        let size = stat.st_size as usize;
-
-        let index_size = size - header_size;
         let index_count = index_size / 40;
         if index_count * 40 != index_size {
             bail!("got unexpected file size");

src/backup/fixed_index.rs

@@ -68,6 +68,19 @@ impl FixedIndexReader {
         file.seek(SeekFrom::Start(0))?;
 
         let header_size = std::mem::size_of::<FixedIndexHeader>();
+
+        let rawfd = file.as_raw_fd();
+        let stat = match nix::sys::stat::fstat(rawfd) {
+            Ok(stat) => stat,
+            Err(err) => bail!("fstat failed - {}", err),
+        };
+
+        let size = stat.st_size as usize;
+
+        if size < header_size {
+            bail!("index too small ({})", stat.st_size);
+        }
+
         let header: Box<FixedIndexHeader> = unsafe { file.read_host_value_boxed()? };
 
         if header.magic != super::FIXED_SIZED_CHUNK_INDEX_1_0 {
@@ -81,12 +94,6 @@ impl FixedIndexReader {
         let index_length = ((size + chunk_size - 1) / chunk_size) as usize;
         let index_size = index_length * 32;
 
-        let rawfd = file.as_raw_fd();
-
-        let stat = match nix::sys::stat::fstat(rawfd) {
-            Ok(stat) => stat,
-            Err(err) => bail!("fstat failed - {}", err),
-        };
 
         let expected_index_size = (stat.st_size as usize) - header_size;
         if index_size != expected_index_size {

src/backup/verify.rs

@@ -482,7 +482,7 @@ pub fn verify_backup_group(
     Ok((count, errors))
 }
 
-/// Verify all backups inside a datastore
+/// Verify all (owned) backups inside a datastore
 ///
 /// Errors are logged to the worker log.
 ///
@@ -493,14 +493,41 @@ pub fn verify_all_backups(
     datastore: Arc<DataStore>,
     worker: Arc<dyn TaskState + Send + Sync>,
     upid: &UPID,
+    owner: Option<Authid>,
     filter: Option<&dyn Fn(&BackupManifest) -> bool>,
 ) -> Result<Vec<String>, Error> {
     let mut errors = Vec::new();
 
+    if let Some(owner) = &owner {
+        task_log!(
+            worker,
+            "verify datastore {} - limiting to backups owned by {}",
+            datastore.name(),
+            owner
+        );
+    }
+
+    let filter_by_owner = |group: &BackupGroup| {
+        if let Some(owner) = &owner {
+            match datastore.get_owner(group) {
+                Ok(ref group_owner) => {
+                    group_owner == owner
+                        || (group_owner.is_token()
+                            && !owner.is_token()
+                            && group_owner.user() == owner.user())
+                },
+                Err(_) => false,
+            }
+        } else {
+            true
+        }
+    };
+
     let mut list = match BackupGroup::list_groups(&datastore.base_path()) {
         Ok(list) => list
             .into_iter()
             .filter(|group| !(group.backup_type() == "host" && group.backup_id() == "benchmark"))
+            .filter(filter_by_owner)
             .collect::<Vec<BackupGroup>>(),
         Err(err) => {
             task_log!(

src/bin/proxmox-backup-api.rs

@@ -52,7 +52,9 @@ async fn run() -> Result<(), Error> {
     let mut config = server::ApiConfig::new(
         buildcfg::JS_DIR, &proxmox_backup::api2::ROUTER, RpcEnvironmentType::PRIVILEGED)?;
 
-    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN)?;
+    let mut commando_sock = server::CommandoSocket::new(server::our_ctrl_sock());
+
+    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN, &mut commando_sock)?;
 
     let rest_server = RestServer::new(config);
 
@@ -76,10 +78,12 @@ async fn run() -> Result<(), Error> {
         },
     );
 
+    server::write_pid(buildcfg::PROXMOX_BACKUP_API_PID_FN)?;
     daemon::systemd_notify(daemon::SystemdNotify::Ready)?;
 
     let init_result: Result<(), Error> = try_block!({
-        server::create_task_control_socket()?;
+        server::register_task_control_commands(&mut commando_sock)?;
+        commando_sock.spawn()?;
         server::server_state_init()?;
         Ok(())
     });

src/bin/proxmox-backup-client.rs

@@ -32,11 +32,11 @@ use proxmox::{
 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
 
 use proxmox_backup::tools;
+use proxmox_backup::api2::access::user::UserWithTokens;
 use proxmox_backup::api2::types::*;
 use proxmox_backup::api2::version;
 use proxmox_backup::client::*;
 use proxmox_backup::pxar::catalog::*;
-use proxmox_backup::config::user::complete_userid;
 use proxmox_backup::backup::{
     archive_type,
     decrypt_key,
@@ -1904,6 +1904,33 @@ fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<Stri
     result
 }
 
+fn complete_auth_id(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
+    proxmox_backup::tools::runtime::main(async { complete_auth_id_do(param).await })
+}
+
+async fn complete_auth_id_do(param: &HashMap<String, String>) -> Vec<String> {
+
+    let mut result = vec![];
+
+    let repo = match extract_repository_from_map(param) {
+        Some(v) => v,
+        _ => return result,
+    };
+
+    let data = try_get(&repo, "api2/json/access/users?include_tokens=true").await;
+
+    if let Ok(parsed) = serde_json::from_value::<Vec<UserWithTokens>>(data) {
+        for user in parsed {
+            result.push(user.userid.to_string());
+            for token in user.tokens {
+                result.push(token.tokenid.to_string());
+            }
+        }
+    };
+
+    result
+}
+
 use proxmox_backup::client::RemoteChunkReader;
 /// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
 /// async use!
@@ -2013,7 +2040,7 @@ fn main() {
     let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
         .arg_param(&["group", "new-owner"])
         .completion_cb("group", complete_backup_group)
-        .completion_cb("new-owner",  complete_userid)
+        .completion_cb("new-owner",  complete_auth_id)
         .completion_cb("repository", complete_repository);
 
     let cmd_def = CliCommandMap::new()

src/bin/proxmox-backup-manager.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::io::{self, Write};
 
 use anyhow::{format_err, Error};
 use serde_json::{json, Value};
@@ -354,6 +355,14 @@ async fn verify(
     Ok(Value::Null)
 }
 
+#[api()]
+/// System report
+async fn report() -> Result<Value, Error> {
+    let report = proxmox_backup::server::generate_report();
+    io::stdout().write_all(report.as_bytes())?;
+    Ok(Value::Null)
+}
+
 fn main() {
 
     proxmox_backup::tools::setup_safe_path_env();
@@ -368,6 +377,7 @@ fn main() {
         .insert("remote", remote_commands())
         .insert("garbage-collection", garbage_collection_commands())
         .insert("cert", cert_mgmt_cli())
+        .insert("subscription", subscription_commands())
         .insert("sync-job", sync_job_commands())
         .insert("task", task_mgmt_cli())
         .insert(
@@ -383,6 +393,9 @@ fn main() {
             CliCommand::new(&API_METHOD_VERIFY)
                 .arg_param(&["store"])
                 .completion_cb("store", config::datastore::complete_datastore_name)
+        )
+        .insert("report",
+            CliCommand::new(&API_METHOD_REPORT)
         );
 
 
@@ -400,29 +413,13 @@ pub fn complete_remote_datastore_name(_arg: &str, param: &HashMap<String, String
 
     let _ = proxmox::try_block!({
         let remote = param.get("remote").ok_or_else(|| format_err!("no remote"))?;
-        let (remote_config, _digest) = config::remote::config()?;
 
-        let remote: config::remote::Remote = remote_config.lookup("remote", &remote)?;
+        let data = crate::tools::runtime::block_on(async move {
+            crate::api2::config::remote::scan_remote_datastores(remote.clone()).await
+        })?;
 
-        let options = HttpClientOptions::new()
+        for item in data {
-            .password(Some(remote.password.clone()))
+            list.push(item.store);
-            .fingerprint(remote.fingerprint.clone());
-
-        let client = HttpClient::new(
-            &remote.host,
-            remote.port.unwrap_or(8007),
-            &remote.userid,
-            options,
-        )?;
-
-        let result = crate::tools::runtime::block_on(client.get("api2/json/admin/datastore", None))?;
-
-        if let Some(data) = result["data"].as_array() {
-            for item in data {
-                if let Some(store) = item["store"].as_str() {
-                    list.push(store.to_owned());
-                }
-            }
         }
 
         Ok(())

src/bin/proxmox-backup-proxy.rs

@@ -1,4 +1,4 @@
-use std::sync::{Arc};
+use std::sync::Arc;
 use std::path::{Path, PathBuf};
 use std::os::unix::io::AsRawFd;
 
@@ -13,7 +13,6 @@ use proxmox::api::RpcEnvironmentType;
 use proxmox_backup::{
     backup::DataStore,
     server::{
-        UPID,
         WorkerTask,
         ApiConfig,
         rest::*,
@@ -30,7 +29,7 @@ use proxmox_backup::{
 };
 
 
-use proxmox_backup::api2::types::{Authid, Userid};
+use proxmox_backup::api2::types::Authid;
 use proxmox_backup::configdir;
 use proxmox_backup::buildcfg;
 use proxmox_backup::server;
@@ -41,6 +40,7 @@ use proxmox_backup::tools::{
         DiskManage,
         zfs_pool_stats,
     },
+    logrotate::LogRotate,
     socket::{
         set_tcp_keepalive,
         PROXMOX_BACKUP_TCP_KEEPALIVE_TIME,
@@ -49,6 +49,7 @@ use proxmox_backup::tools::{
 
 use proxmox_backup::api2::pull::do_sync_job;
 use proxmox_backup::server::do_verification_job;
+use proxmox_backup::server::do_prune_job;
 
 fn main() -> Result<(), Error> {
     proxmox_backup::tools::setup_safe_path_env();
@@ -73,6 +74,10 @@ async fn run() -> Result<(), Error> {
         bail!("unable to inititialize syslog - {}", err);
     }
 
+    // Note: To debug early connection error use
+    // PROXMOX_DEBUG=1 ./target/release/proxmox-backup-proxy
+    let debug = std::env::var("PROXMOX_DEBUG").is_ok();
+
     let _ = public_auth_key(); // load with lazy_static
     let _ = csrf_secret(); // load with lazy_static
 
@@ -85,7 +90,6 @@ async fn run() -> Result<(), Error> {
     config.add_alias("xtermjs", "/usr/share/pve-xtermjs");
     config.add_alias("locale", "/usr/share/pbs-i18n");
     config.add_alias("widgettoolkit", "/usr/share/javascript/proxmox-widget-toolkit");
-    config.add_alias("css", "/usr/share/javascript/proxmox-backup/css");
     config.add_alias("docs", "/usr/share/doc/proxmox-backup/html");
 
     let mut indexpath = PathBuf::from(buildcfg::JS_DIR);
@@ -93,7 +97,9 @@ async fn run() -> Result<(), Error> {
     config.register_template("index", &indexpath)?;
     config.register_template("console", "/usr/share/pve-xtermjs/index.html.hbs")?;
 
-    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN)?;
+    let mut commando_sock = server::CommandoSocket::new(server::our_ctrl_sock());
+
+    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN, &mut commando_sock)?;
 
     let rest_server = RestServer::new(config);
 
@@ -113,25 +119,12 @@ async fn run() -> Result<(), Error> {
     let server = daemon::create_daemon(
         ([0,0,0,0,0,0,0,0], 8007).into(),
         |listener, ready| {
-            let connections = proxmox_backup::tools::async_io::StaticIncoming::from(listener)
-                .map_err(Error::from)
-                .try_filter_map(move |(sock, _addr)| {
-                    let acceptor = Arc::clone(&acceptor);
-                    async move {
-                        sock.set_nodelay(true).unwrap();
 
-                        let _ = set_tcp_keepalive(sock.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME);
+            let connections = accept_connections(listener, acceptor, debug);
-
+            let connections = hyper::server::accept::from_stream(connections);
-                        Ok(tokio_openssl::accept(&acceptor, sock)
-                            .await
-                            .ok() // handshake errors aren't be fatal, so return None to filter
-                        )
-                    }
-                });
-            let connections = proxmox_backup::tools::async_io::HyperAccept(connections);
 
             Ok(ready
-                .and_then(|_| hyper::Server::builder(connections)
+               .and_then(|_| hyper::Server::builder(connections)
                     .serve(rest_server)
                     .with_graceful_shutdown(server::shutdown_future())
                     .map_err(Error::from)
@@ -142,10 +135,12 @@ async fn run() -> Result<(), Error> {
         },
     );
 
+    server::write_pid(buildcfg::PROXMOX_BACKUP_PROXY_PID_FN)?;
     daemon::systemd_notify(daemon::SystemdNotify::Ready)?;
 
     let init_result: Result<(), Error> = try_block!({
-        server::create_task_control_socket()?;
+        server::register_task_control_commands(&mut commando_sock)?;
+        commando_sock.spawn()?;
         server::server_state_init()?;
         Ok(())
     });
@@ -165,6 +160,72 @@ async fn run() -> Result<(), Error> {
     Ok(())
 }
 
+fn accept_connections(
+    mut listener: tokio::net::TcpListener,
+    acceptor: Arc<openssl::ssl::SslAcceptor>,
+    debug: bool,
+) -> tokio::sync::mpsc::Receiver<Result<tokio_openssl::SslStream<tokio::net::TcpStream>, Error>> {
+
+    const MAX_PENDING_ACCEPTS: usize = 1024;
+
+    let (sender, receiver) = tokio::sync::mpsc::channel(MAX_PENDING_ACCEPTS);
+
+    let accept_counter = Arc::new(());
+
+    tokio::spawn(async move {
+        loop {
+            match listener.accept().await {
+                Err(err) => {
+                    eprintln!("error accepting tcp connection: {}", err);
+                }
+                Ok((sock, _addr)) =>  {
+                    sock.set_nodelay(true).unwrap();
+                    let _ = set_tcp_keepalive(sock.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME);
+                    let acceptor = Arc::clone(&acceptor);
+                    let mut sender = sender.clone();
+
+                    if Arc::strong_count(&accept_counter) > MAX_PENDING_ACCEPTS {
+                        eprintln!("connection rejected - to many open connections");
+                        continue;
+                    }
+
+                    let accept_counter = accept_counter.clone();
+                    tokio::spawn(async move {
+                        let accept_future = tokio::time::timeout(
+                            Duration::new(10, 0), tokio_openssl::accept(&acceptor, sock));
+
+                        let result = accept_future.await;
+
+                        match result {
+                            Ok(Ok(connection)) => {
+                                if let Err(_) = sender.send(Ok(connection)).await {
+                                    if debug {
+                                        eprintln!("detect closed connection channel");
+                                    }
+                                }
+                            }
+                            Ok(Err(err)) => {
+                                if debug {
+                                    eprintln!("https handshake failed - {}", err);
+                                }
+                            }
+                            Err(_) => {
+                                if debug {
+                                    eprintln!("https handshake timeout");
+                                }
+                            }
+                        }
+
+                        drop(accept_counter); // decrease reference count
+                    });
+                }
+            }
+        }
+    });
+
+    receiver
+}
+
 fn start_stat_generator() {
     let abort_future = server::shutdown_future();
     let future = Box::pin(run_stat_generator());
@@ -247,8 +308,6 @@ async fn schedule_datastore_garbage_collection() {
         },
     };
 
-    let email = server::lookup_user_email(Userid::root_userid());
-
     let config = match datastore::config() {
         Err(err) => {
             eprintln!("unable to read datastore config - {}", err);
@@ -291,22 +350,11 @@ async fn schedule_datastore_garbage_collection() {
 
         let worker_type = "garbage_collection";
 
-        let stat = datastore.last_gc_status();
+        let last = match jobstate::last_run_time(worker_type, &store) {
-        let last = if let Some(upid_str) = stat.upid {
+            Ok(time) => time,
-            match upid_str.parse::<UPID>() {
+            Err(err) => {
-                Ok(upid) => upid.starttime,
+                eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
-                Err(err) => {
+                continue;
-                    eprintln!("unable to parse upid '{}' - {}", upid_str, err);
-                    continue;
-                }
-            }
-        } else {
-            match jobstate::last_run_time(worker_type, &store) {
-                Ok(time) => time,
-                Err(err) => {
-                    eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
-                    continue;
-                }
             }
         };
 
@@ -323,44 +371,15 @@ async fn schedule_datastore_garbage_collection() {
 
         if next > now  { continue; }
 
-        let mut job = match Job::new(worker_type, &store) {
+        let job = match Job::new(worker_type, &store) {
             Ok(job) => job,
             Err(_) => continue, // could not get lock
         };
 
-        let store2 = store.clone();
+        let auth_id = Authid::root_auth_id();
-        let email2 = email.clone();
 
-        if let Err(err) = WorkerTask::new_thread(
+        if let Err(err) = crate::server::do_garbage_collection_job(job, datastore, auth_id, Some(event_str), false) {
-            worker_type,
+            eprintln!("unable to start garbage collection job on datastore {} - {}", store, err);
-            Some(store.clone()),
-            Authid::backup_auth_id().clone(),
-            false,
-            move |worker| {
-                job.start(&worker.upid().to_string())?;
-
-                worker.log(format!("starting garbage collection on store {}", store));
-                worker.log(format!("task triggered by schedule '{}'", event_str));
-
-                let result = datastore.garbage_collection(&*worker, worker.upid());
-
-                let status = worker.create_state(&result);
-
-                if let Err(err) = job.finish(status) {
-                    eprintln!("could not finish job state for {}: {}", worker_type, err);
-                }
-
-                if let Some(email2) = email2 {
-                    let gc_status = datastore.last_gc_status();
-                    if let Err(err) = crate::server::send_gc_status(&email2, datastore.name(), &gc_status, &result) {
-                        eprintln!("send gc notification failed: {}", err);
-                    }
-                }
-
-                result
-            }
-        ) {
-            eprintln!("unable to start garbage collection on store {} - {}", store2, err);
         }
     }
 }
@@ -370,8 +389,6 @@ async fn schedule_datastore_prune() {
     use proxmox_backup::{
         backup::{
             PruneOptions,
-            BackupGroup,
-            compute_prune_info,
         },
         config::datastore::{
             self,
@@ -388,13 +405,6 @@ async fn schedule_datastore_prune() {
     };
 
     for (store, (_, store_config)) in config.sections {
-        let datastore = match DataStore::lookup_datastore(&store) {
-            Ok(datastore) => datastore,
-            Err(err) => {
-                eprintln!("lookup_datastore '{}' failed - {}", store, err);
-                continue;
-            }
-        };
 
         let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
             Ok(c) => c,
@@ -422,95 +432,18 @@ async fn schedule_datastore_prune() {
             continue;
         }
 
-        let event = match parse_calendar_event(&event_str) {
-            Ok(event) => event,
-            Err(err) => {
-                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
-                continue;
-            }
-        };
-
         let worker_type = "prune";
+        if check_schedule(worker_type, &event_str, &store) {
+            let job = match Job::new(worker_type, &store) {
+                Ok(job) => job,
+                Err(_) => continue, // could not get lock
+            };
 
-        let last = match jobstate::last_run_time(worker_type, &store) {
+            let auth_id = Authid::root_auth_id().clone();
-            Ok(time) => time,
+            if let Err(err) = do_prune_job(job, prune_options, store.clone(), &auth_id, Some(event_str)) {
-            Err(err) => {
+                eprintln!("unable to start datastore prune job {} - {}", &store, err);
-                eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
-                continue;
             }
         };
-
-        let next = match compute_next_event(&event, last, false) {
-            Ok(Some(next)) => next,
-            Ok(None) => continue,
-            Err(err) => {
-                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
-                continue;
-            }
-        };
-
-        let now = proxmox::tools::time::epoch_i64();
-
-        if next > now  { continue; }
-
-        let mut job = match Job::new(worker_type, &store) {
-            Ok(job) => job,
-            Err(_) => continue, // could not get lock
-        };
-
-        let store2 = store.clone();
-
-        if let Err(err) = WorkerTask::new_thread(
-            worker_type,
-            Some(store.clone()),
-            Authid::backup_auth_id().clone(),
-            false,
-            move |worker| {
-
-                job.start(&worker.upid().to_string())?;
-
-                let result = try_block!({
-
-                    worker.log(format!("Starting datastore prune on store \"{}\"", store));
-                    worker.log(format!("task triggered by schedule '{}'", event_str));
-                    worker.log(format!("retention options: {}", prune_options.cli_options_string()));
-
-                    let base_path = datastore.base_path();
-
-                    let groups = BackupGroup::list_groups(&base_path)?;
-                    for group in groups {
-                        let list = group.list_backups(&base_path)?;
-                        let mut prune_info = compute_prune_info(list, &prune_options)?;
-                        prune_info.reverse(); // delete older snapshots first
-
-                        worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
-                                store, group.backup_type(), group.backup_id()));
-
-                        for (info, keep) in prune_info {
-                            worker.log(format!(
-                                    "{} {}/{}/{}",
-                                    if keep { "keep" } else { "remove" },
-                                    group.backup_type(), group.backup_id(),
-                                    info.backup_dir.backup_time_string()));
-                            if !keep {
-                                datastore.remove_backup_dir(&info.backup_dir, true)?;
-                            }
-                        }
-                    }
-                    Ok(())
-                });
-
-                let status = worker.create_state(&result);
-
-                if let Err(err) = job.finish(status) {
-                    eprintln!("could not finish job state for {}: {}", worker_type, err);
-                }
-
-                result
-            }
-        ) {
-            eprintln!("unable to start datastore prune on store {} - {}", store2, err);
-        }
     }
 }
 
@@ -543,47 +476,18 @@ async fn schedule_datastore_sync_jobs() {
             None => continue,
         };
 
-        let event = match parse_calendar_event(&event_str) {
-            Ok(event) => event,
-            Err(err) => {
-                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
-                continue;
-            }
-        };
-
         let worker_type = "syncjob";
+        if check_schedule(worker_type, &event_str, &job_id) {
+            let job = match Job::new(worker_type, &job_id) {
+                Ok(job) => job,
+                Err(_) => continue, // could not get lock
+            };
 
-        let last = match jobstate::last_run_time(worker_type, &job_id) {
+            let auth_id = Authid::root_auth_id().clone();
-            Ok(time) => time,
+            if let Err(err) = do_sync_job(job, job_config, &auth_id, Some(event_str)) {
-            Err(err) => {
+                eprintln!("unable to start datastore sync job {} - {}", &job_id, err);
-                eprintln!("could not get last run time of {} {}: {}", worker_type, job_id, err);
-                continue;
             }
         };
-
-        let next = match compute_next_event(&event, last, false) {
-            Ok(Some(next)) => next,
-            Ok(None) => continue,
-            Err(err) => {
-                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
-                continue;
-            }
-        };
-
-        let now = proxmox::tools::time::epoch_i64();
-
-        if next > now  { continue; }
-
-        let job = match Job::new(worker_type, &job_id) {
-            Ok(job) => job,
-            Err(_) => continue, // could not get lock
-        };
-
-        let auth_id = Authid::backup_auth_id();
-
-        if let Err(err) = do_sync_job(job, job_config, &auth_id, Some(event_str)) {
-            eprintln!("unable to start datastore sync job {} - {}", &job_id, err);
-        }
     }
 }
 
@@ -613,79 +517,30 @@ async fn schedule_datastore_verify_jobs() {
             Some(ref event_str) => event_str.clone(),
             None => continue,
         };
-        let event = match parse_calendar_event(&event_str) {
+
-            Ok(event) => event,
-            Err(err) => {
-                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
-                continue;
-            }
-        };
         let worker_type = "verificationjob";
-        let last = match jobstate::last_run_time(worker_type, &job_id) {
+        let auth_id = Authid::root_auth_id().clone();
-            Ok(time) => time,
+        if check_schedule(worker_type, &event_str, &job_id) {
-            Err(err) => {
+            let job = match Job::new(&worker_type, &job_id) {
-                eprintln!("could not get last run time of {} {}: {}", worker_type, job_id, err);
+                Ok(job) => job,
-                continue;
+                Err(_) => continue, // could not get lock
+            };
+            if let Err(err) = do_verification_job(job, job_config, &auth_id, Some(event_str)) {
+                eprintln!("unable to start datastore verification job {} - {}", &job_id, err);
             }
         };
-        let next = match compute_next_event(&event, last, false) {
-            Ok(Some(next)) => next,
-            Ok(None) => continue,
-            Err(err) => {
-                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
-                continue;
-            }
-        };
-        let now = proxmox::tools::time::epoch_i64();
-        if next > now { continue; }
-        let job = match Job::new(worker_type, &job_id) {
-            Ok(job) => job,
-            Err(_) => continue, // could not get lock
-        };
-        let auth_id = Authid::backup_auth_id();
-        if let Err(err) = do_verification_job(job, job_config, &auth_id, Some(event_str)) {
-            eprintln!("unable to start datastore verification job {} - {}", &job_id, err);
-        }
     }
 }
 
 async fn schedule_task_log_rotate() {
 
     let worker_type = "logrotate";
-    let job_id = "task_archive";
+    let job_id = "access-log_and_task-archive";
-
-    let last = match jobstate::last_run_time(worker_type, job_id) {
-        Ok(time) => time,
-        Err(err) => {
-            eprintln!("could not get last run time of task log archive rotation: {}", err);
-            return;
-        }
-    };
 
     // schedule daily at 00:00 like normal logrotate
     let schedule = "00:00";
 
-    let event = match parse_calendar_event(schedule) {
+    if !check_schedule(worker_type, schedule, job_id) {
-        Ok(event) => event,
-        Err(err) => {
-            // should not happen?
-            eprintln!("unable to parse schedule '{}' - {}", schedule, err);
-            return;
-        }
-    };
-
-    let next = match compute_next_event(&event, last, false) {
-        Ok(Some(next)) => next,
-        Ok(None) => return,
-        Err(err) => {
-            eprintln!("compute_next_event for '{}' failed - {}", schedule, err);
-            return;
-        }
-    };
-
-    let now = proxmox::tools::time::epoch_i64();
-
-    if next > now {
         // if we never ran the rotation, schedule instantly
         match jobstate::JobState::load(worker_type, job_id) {
             Ok(state) => match state {
@@ -703,17 +558,16 @@ async fn schedule_task_log_rotate() {
 
     if let Err(err) = WorkerTask::new_thread(
         worker_type,
-        Some(job_id.to_string()),
+        None,
-        Authid::backup_auth_id().clone(),
+        Authid::root_auth_id().clone(),
         false,
         move |worker| {
             job.start(&worker.upid().to_string())?;
             worker.log(format!("starting task log rotation"));
 
             let result = try_block!({
-                // rotate task log archive
+                let max_size = 512 * 1024 - 1; // an entry has ~ 100b, so > 5000 entries/file
-                let max_size = 500000; // a normal entry has about 100b, so ~ 5000 entries/file
+                let max_files = 20; // times twenty files gives > 100000 task entries
-                let max_files = 20; // times twenty files gives at least 100000 task entries
                 let has_rotated = rotate_task_log_archive(max_size, true, Some(max_files))?;
                 if has_rotated {
                     worker.log(format!("task log archive was rotated"));
@@ -721,6 +575,28 @@ async fn schedule_task_log_rotate() {
                     worker.log(format!("task log archive was not rotated"));
                 }
 
+                let max_size = 32 * 1024 * 1024 - 1;
+                let max_files = 14;
+                let mut logrotate = LogRotate::new(buildcfg::API_ACCESS_LOG_FN, true)
+                        .ok_or_else(|| format_err!("could not get API access log file names"))?;
+
+                if logrotate.rotate(max_size, None, Some(max_files))? {
+                    println!("rotated access log, telling daemons to re-open log file");
+                    proxmox_backup::tools::runtime::block_on(command_reopen_logfiles())?;
+                    worker.log(format!("API access log was rotated"));
+                } else {
+                    worker.log(format!("API access log was not rotated"));
+                }
+
+                let mut logrotate = LogRotate::new(buildcfg::API_AUTH_LOG_FN, true)
+                        .ok_or_else(|| format_err!("could not get API auth log file names"))?;
+
+                if logrotate.rotate(max_size, None, Some(max_files))? {
+                    worker.log(format!("API access log was rotated"));
+                } else {
+                    worker.log(format!("API access log was not rotated"));
+                }
+
                 Ok(())
             });
 
@@ -738,6 +614,28 @@ async fn schedule_task_log_rotate() {
 
 }
 
+async fn command_reopen_logfiles() -> Result<(), Error> {
+    // only care about the most recent daemon instance for each, proxy & api, as other older ones
+    // should not respond to new requests anyway, but only finish their current one and then exit.
+    let sock = server::our_ctrl_sock();
+    let f1 = server::send_command(sock, serde_json::json!({
+        "command": "api-access-log-reopen",
+    }));
+
+    let pid = server::read_pid(buildcfg::PROXMOX_BACKUP_API_PID_FN)?;
+    let sock = server::ctrl_sock_from_pid(pid);
+    let f2 = server::send_command(sock, serde_json::json!({
+        "command": "api-access-log-reopen",
+    }));
+
+    match futures::join!(f1, f2) {
+        (Err(e1), Err(e2)) => Err(format_err!("reopen commands failed, proxy: {}; api: {}", e1, e2)),
+        (Err(e1), Ok(_)) => Err(format_err!("reopen commands failed, proxy: {}", e1)),
+        (Ok(_), Err(e2)) => Err(format_err!("reopen commands failed, api: {}", e2)),
+        _ => Ok(()),
+    }
+}
+
 async fn run_stat_generator() {
 
     let mut count = 0;
@@ -850,6 +748,36 @@ async fn generate_host_stats(save: bool) {
     });
 }
 
+fn check_schedule(worker_type: &str, event_str: &str, id: &str) -> bool {
+    let event = match parse_calendar_event(event_str) {
+        Ok(event) => event,
+        Err(err) => {
+            eprintln!("unable to parse schedule '{}' - {}", event_str, err);
+            return false;
+        }
+    };
+
+    let last = match jobstate::last_run_time(worker_type, &id) {
+        Ok(time) => time,
+        Err(err) => {
+            eprintln!("could not get last run time of {} {}: {}", worker_type, id, err);
+            return false;
+        }
+    };
+
+    let next = match compute_next_event(&event, last, false) {
+        Ok(Some(next)) => next,
+        Ok(None) => return false,
+        Err(err) => {
+            eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
+            return false;
+        }
+    };
+
+    let now = proxmox::tools::time::epoch_i64();
+    next <= now
+}
+
 fn gather_disk_stats(disk_manager: Arc<DiskManage>, path: &Path, rrd_prefix: &str, save: bool) {
 
     match proxmox_backup::tools::disks::disk_usage(path) {

src/bin/proxmox-daily-update.rs

@@ -0,0 +1,73 @@
+use anyhow::Error;
+use serde_json::{json, Value};
+
+use proxmox::api::{cli::*, RpcEnvironment, ApiHandler};
+
+use proxmox_backup::api2;
+use proxmox_backup::tools::subscription;
+
+async fn wait_for_local_worker(upid_str: &str) -> Result<(), Error> {
+    let upid: proxmox_backup::server::UPID = upid_str.parse()?;
+    let sleep_duration = core::time::Duration::new(0, 100_000_000);
+
+    loop {
+        if !proxmox_backup::server::worker_is_active_local(&upid) {
+            break;
+        }
+        tokio::time::delay_for(sleep_duration).await;
+    }
+    Ok(())
+}
+
+/// Daily update
+async fn do_update(
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Value, Error> {
+    let param = json!({});
+
+    let method = &api2::node::subscription::API_METHOD_CHECK_SUBSCRIPTION;
+    let _res = match method.handler {
+        ApiHandler::Sync(handler) => (handler)(param, method, rpcenv)?,
+        _ => unreachable!(),
+    };
+
+    let notify = match subscription::read_subscription() {
+        Ok(Some(subscription)) => subscription.status == subscription::SubscriptionStatus::ACTIVE,
+        Ok(None) => false,
+        Err(err) => {
+            eprintln!("Error reading subscription - {}", err);
+            false
+        },
+    };
+
+    let param = json!({
+        "notify": notify,
+    });
+    let method = &api2::node::apt::API_METHOD_APT_UPDATE_DATABASE;
+    let upid = match method.handler {
+        ApiHandler::Sync(handler) => (handler)(param, method, rpcenv)?,
+        _ => unreachable!(),
+    };
+    wait_for_local_worker(upid.as_str().unwrap()).await?;
+
+    // TODO: certificate checks/renewal/... ?
+
+    // TODO: cleanup tasks like in PVE?
+
+    Ok(Value::Null)
+}
+
+fn main() {
+    proxmox_backup::tools::setup_safe_path_env();
+
+    let mut rpcenv = CliEnvironment::new();
+    rpcenv.set_auth_id(Some(String::from("root@pam")));
+
+    match proxmox_backup::tools::runtime::main(do_update(&mut rpcenv)) {
+        Err(err) => {
+            eprintln!("error during update: {}", err);
+            std::process::exit(1);
+        },
+        _ => (),
+    }
+}

src/bin/proxmox_backup_manager/acl.rs

@@ -60,7 +60,7 @@ pub fn acl_commands() -> CommandLineInterface {
             "update",
             CliCommand::new(&api2::access::acl::API_METHOD_UPDATE_ACL)
                 .arg_param(&["path", "role"])
-                .completion_cb("userid", config::user::complete_userid)
+                .completion_cb("auth-id", config::user::complete_authid)
                 .completion_cb("path", config::datastore::complete_acl_path)
 
         );

src/bin/proxmox_backup_manager/mod.rs

@@ -14,5 +14,7 @@ mod sync;
 pub use sync::*;
 mod user;
 pub use user::*;
+mod subscription;
+pub use subscription::*;
 mod disk;
 pub use disk::*;

src/bin/proxmox_backup_manager/subscription.rs

@@ -0,0 +1,55 @@
+use anyhow::Error;
+use serde_json::Value;
+
+use proxmox::api::{api, cli::*, RpcEnvironment, ApiHandler};
+
+use proxmox_backup::api2;
+
+#[api(
+    input: {
+        properties: {
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Read subscription info.
+fn get(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+
+    let output_format = get_output_format(&param);
+
+    let info = &api2::node::subscription::API_METHOD_GET_SUBSCRIPTION;
+    let mut data = match info.handler {
+        ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
+        _ => unreachable!(),
+    };
+
+    let options = default_table_format_options();
+    format_and_print_result_full(&mut data, info.returns, &output_format, &options);
+
+    Ok(Value::Null)
+}
+
+pub fn subscription_commands() -> CommandLineInterface {
+
+    let cmd_def = CliCommandMap::new()
+        .insert("get", CliCommand::new(&API_METHOD_GET))
+        .insert("set",
+            CliCommand::new(&api2::node::subscription::API_METHOD_SET_SUBSCRIPTION)
+                .fixed_param("node", "localhost".into())
+                .arg_param(&["key"])
+        )
+        .insert("update",
+            CliCommand::new(&api2::node::subscription::API_METHOD_CHECK_SUBSCRIPTION)
+                .fixed_param("node", "localhost".into())
+        )
+        .insert("remove",
+            CliCommand::new(&api2::node::subscription::API_METHOD_DELETE_SUBSCRIPTION)
+                .fixed_param("node", "localhost".into())
+        )
+        ;
+
+    cmd_def.into()
+}

src/bin/proxmox_backup_manager/user.rs

@@ -100,7 +100,7 @@ fn list_tokens(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, E
                 schema: OUTPUT_FORMAT,
                 optional: true,
             },
-            auth_id: {
+            "auth-id": {
                 type: Authid,
             },
             path: {
@@ -195,8 +195,8 @@ pub fn user_commands() -> CommandLineInterface {
         .insert(
             "permissions",
             CliCommand::new(&&API_METHOD_LIST_PERMISSIONS)
-                .arg_param(&["auth_id"])
+                .arg_param(&["auth-id"])
-                .completion_cb("auth_id", config::user::complete_authid)
+                .completion_cb("auth-id", config::user::complete_authid)
                 .completion_cb("path", config::datastore::complete_acl_path)
         );
 

src/buildcfg.rs

@@ -4,7 +4,31 @@
 pub const CONFIGDIR: &str = "/etc/proxmox-backup";
 pub const JS_DIR: &str = "/usr/share/javascript/proxmox-backup";
 
-pub const API_ACCESS_LOG_FN: &str = "/var/log/proxmox-backup/api/access.log";
+#[macro_export]
+macro_rules! PROXMOX_BACKUP_RUN_DIR_M { () => ("/run/proxmox-backup") }
+
+#[macro_export]
+macro_rules! PROXMOX_BACKUP_LOG_DIR_M { () => ("/var/log/proxmox-backup") }
+
+/// namespaced directory for in-memory (tmpfs) run state
+pub const PROXMOX_BACKUP_RUN_DIR: &str = PROXMOX_BACKUP_RUN_DIR_M!();
+
+/// namespaced directory for persistent logging
+pub const PROXMOX_BACKUP_LOG_DIR: &str = PROXMOX_BACKUP_LOG_DIR_M!();
+
+/// logfile for all API reuests handled by the proxy and privileged API daemons. Note that not all
+/// failed logins can be logged here with full information, use the auth log for that.
+pub const API_ACCESS_LOG_FN: &str = concat!(PROXMOX_BACKUP_LOG_DIR_M!(), "/api/access.log");
+
+/// logfile for any failed authentication, via ticket or via token, and new successfull ticket
+/// creations. This file can be useful for fail2ban.
+pub const API_AUTH_LOG_FN: &str = concat!(PROXMOX_BACKUP_LOG_DIR_M!(), "/api/auth.log");
+
+/// the PID filename for the unprivileged proxy daemon
+pub const PROXMOX_BACKUP_PROXY_PID_FN: &str = concat!(PROXMOX_BACKUP_RUN_DIR_M!(), "/proxy.pid");
+
+/// the PID filename for the privileged api daemon
+pub const PROXMOX_BACKUP_API_PID_FN: &str = concat!(PROXMOX_BACKUP_RUN_DIR_M!(), "/api.pid");
 
 /// Prepend configuration directory to a file name
 ///

src/client/http_client.rs

@@ -405,6 +405,9 @@ impl HttpClient {
     ///
     /// Login is done on demand, so this is only required if you need
     /// access to authentication data in 'AuthInfo'.
+    ///
+    /// Note: tickets a periodially re-newed, so one can use this
+    /// to query changed ticket.
     pub async fn login(&self) -> Result<AuthInfo, Error> {
         if let Some(future) = &self.first_auth {
             future.listen().await?;
@@ -493,7 +496,7 @@ impl HttpClient {
 
         let auth =  self.login().await?;
         if auth.auth_id.is_token() {
-            let enc_api_token = format!("{}:{}", auth.auth_id, percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET));
+            let enc_api_token = format!("PBSAPIToken {}:{}", auth.auth_id, percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET));
             req.headers_mut().insert("Authorization", HeaderValue::from_str(&enc_api_token).unwrap());
         } else {
             let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET));
@@ -602,7 +605,7 @@ impl HttpClient {
         let auth =  self.login().await?;
 
         if auth.auth_id.is_token() {
-            let enc_api_token = format!("{}:{}", auth.auth_id, percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET));
+            let enc_api_token = format!("PBSAPIToken {}:{}", auth.auth_id, percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET));
             req.headers_mut().insert("Authorization", HeaderValue::from_str(&enc_api_token).unwrap());
         } else {
             let enc_ticket = format!("PBSAuthCookie={}", percent_encode(auth.ticket.as_bytes(), DEFAULT_ENCODE_SET));

src/client/pull.rs

@@ -103,7 +103,7 @@ async fn pull_index_chunks<I: IndexFile>(
 
     let bytes = bytes.load(Ordering::SeqCst);
 
-    worker.log(format!("downloaded {} bytes ({} MiB/s)", bytes, (bytes as f64)/(1024.0*1024.0*elapsed)));
+    worker.log(format!("downloaded {} bytes ({:.2} MiB/s)", bytes, (bytes as f64)/(1024.0*1024.0*elapsed)));
 
     Ok(())
 }
@@ -410,7 +410,8 @@ pub async fn pull_group(
 
     list.sort_unstable_by(|a, b| a.backup_time.cmp(&b.backup_time));
 
-    let auth_info = client.login().await?;
+    client.login().await?; // make sure auth is complete
+
     let fingerprint = client.fingerprint();
 
     let last_sync = tgt_store.last_successful_backup(group)?;
@@ -447,6 +448,9 @@ pub async fn pull_group(
             if last_sync_time > backup_time { continue; }
         }
 
+        // get updated auth_info (new tickets)
+        let auth_info = client.login().await?;
+
         let options = HttpClientOptions::new()
             .password(Some(auth_info.ticket.clone()))
             .fingerprint(fingerprint.clone());

src/config/acl.rs

@@ -26,14 +26,23 @@ constnamedbitmap! {
         PRIV_SYS_MODIFY("Sys.Modify");
         PRIV_SYS_POWER_MANAGEMENT("Sys.PowerManagement");
 
+        /// Datastore.Audit allows knowing about a datastore,
+        /// including reading the configuration entry and listing its contents
         PRIV_DATASTORE_AUDIT("Datastore.Audit");
+        /// Datastore.Allocate allows creating or deleting datastores
         PRIV_DATASTORE_ALLOCATE("Datastore.Allocate");
+        /// Datastore.Modify allows modifying a datastore and its contents
         PRIV_DATASTORE_MODIFY("Datastore.Modify");
+        /// Datastore.Read allows reading arbitrary backup contents
         PRIV_DATASTORE_READ("Datastore.Read");
+        /// Allows verifying a datastore
+        PRIV_DATASTORE_VERIFY("Datastore.Verify");
 
-        /// Datastore.Backup also requires backup ownership
+        /// Datastore.Backup allows Datastore.Read|Verify and creating new snapshots,
+        /// but also requires backup ownership
         PRIV_DATASTORE_BACKUP("Datastore.Backup");
-        /// Datastore.Prune also requires backup ownership
+        /// Datastore.Prune allows deleting snapshots,
+        /// but also requires backup ownership
         PRIV_DATASTORE_PRUNE("Datastore.Prune");
 
         PRIV_PERMISSIONS_MODIFY("Permissions.Modify");
@@ -41,7 +50,6 @@ constnamedbitmap! {
         PRIV_REMOTE_AUDIT("Remote.Audit");
         PRIV_REMOTE_MODIFY("Remote.Modify");
         PRIV_REMOTE_READ("Remote.Read");
-        PRIV_REMOTE_PRUNE("Remote.Prune");
 
         PRIV_SYS_CONSOLE("Sys.Console");
     }
@@ -64,12 +72,14 @@ pub const ROLE_DATASTORE_ADMIN: u64 =
 PRIV_DATASTORE_AUDIT |
 PRIV_DATASTORE_MODIFY |
 PRIV_DATASTORE_READ |
+PRIV_DATASTORE_VERIFY |
 PRIV_DATASTORE_BACKUP |
 PRIV_DATASTORE_PRUNE;
 
-/// Datastore.Reader can read datastore content an do restore
+/// Datastore.Reader can read/verify datastore content and do restore
 pub const ROLE_DATASTORE_READER: u64 =
 PRIV_DATASTORE_AUDIT |
+PRIV_DATASTORE_VERIFY |
 PRIV_DATASTORE_READ;
 
 /// Datastore.Backup can do backup and restore, but no prune.
@@ -93,14 +103,12 @@ PRIV_REMOTE_AUDIT;
 pub const ROLE_REMOTE_ADMIN: u64 =
 PRIV_REMOTE_AUDIT |
 PRIV_REMOTE_MODIFY |
-PRIV_REMOTE_READ |
+PRIV_REMOTE_READ;
-PRIV_REMOTE_PRUNE;
 
 /// Remote.SyncOperator can do read and prune on the remote.
 pub const ROLE_REMOTE_SYNC_OPERATOR: u64 =
 PRIV_REMOTE_AUDIT |
-PRIV_REMOTE_READ |
+PRIV_REMOTE_READ;
-PRIV_REMOTE_PRUNE;
 
 pub const ROLE_NAME_NO_ACCESS: &str ="NoAccess";
 

src/config/cached_user_info.rs

@@ -57,6 +57,14 @@ impl CachedUserInfo {
         Ok(config)
     }
 
+    #[cfg(test)]
+    pub(crate) fn test_new(user_cfg: SectionConfigData, acl_tree: AclTree) -> Self {
+        Self {
+            user_cfg: Arc::new(user_cfg),
+            acl_tree: Arc::new(acl_tree),
+        }
+    }
+
     /// Test if a authentication id is enabled and not expired
     pub fn is_active_auth_id(&self, auth_id: &Authid) -> bool {
         let userid = auth_id.user();

src/config/datastore.rs

@@ -32,6 +32,14 @@ pub const DIR_NAME_SCHEMA: Schema = StringSchema::new("Directory name").schema()
         path: {
             schema: DIR_NAME_SCHEMA,
         },
+        "notify-user": {
+            optional: true,
+            type: Userid,
+        },
+        "notify": {
+            optional: true,
+            schema: DATASTORE_NOTIFY_STRING_SCHEMA,
+        },
         comment: {
             optional: true,
             schema: SINGLE_LINE_COMMENT_SCHEMA,
@@ -101,6 +109,12 @@ pub struct DataStoreConfig {
     /// If enabled, all backups will be verified right after completion.
     #[serde(skip_serializing_if="Option::is_none")]
     pub verify_new: Option<bool>,
+    /// Send job email notification to this user
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub notify_user: Option<Userid>,
+    /// Send notification only for job errors
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub notify: Option<String>,
 }
 
 fn init() -> SectionConfig {

src/config/network/helper.rs

@@ -57,38 +57,57 @@ lazy_static! {
 }
 
 pub fn parse_cidr(cidr: &str) -> Result<(String, u8, bool), Error> {
+    let (address, mask, is_v6) = parse_address_or_cidr(cidr)?;
+    if let Some(mask) = mask {
+        return Ok((address, mask, is_v6));
+    } else {
+        bail!("missing netmask in '{}'", cidr);
+    }
+}
+
+pub fn check_netmask(mask: u8, is_v6: bool) -> Result<(), Error> {
+    if is_v6 {
+        if !(mask >= 1 && mask <= 128) {
+            bail!("IPv6 mask '{}' is out of range (1..128).", mask);
+        }
+    } else {
+        if !(mask > 0 && mask <= 32) {
+            bail!("IPv4 mask '{}' is out of range (1..32).", mask);
+        }
+    }
+    Ok(())
+}
+
+// parse ip address with otional cidr mask
+pub fn parse_address_or_cidr(cidr: &str) -> Result<(String, Option<u8>, bool), Error> {
 
     lazy_static! {
         pub static ref CIDR_V4_REGEX: Regex = Regex::new(
-            concat!(r"^(", IPV4RE!(), r")(?:/(\d{1,2}))$")
+            concat!(r"^(", IPV4RE!(), r")(?:/(\d{1,2}))?$")
         ).unwrap();
         pub static ref CIDR_V6_REGEX: Regex = Regex::new(
-            concat!(r"^(", IPV6RE!(), r")(?:/(\d{1,3}))$")
+            concat!(r"^(", IPV6RE!(), r")(?:/(\d{1,3}))?$")
         ).unwrap();
     }
 
     if let Some(caps) = CIDR_V4_REGEX.captures(&cidr) {
         let address = &caps[1];
-        let mask = &caps[2];
+        if let Some(mask) = caps.get(2) {
-        let mask = u8::from_str_radix(mask, 10)
+            let mask = u8::from_str_radix(mask.as_str(), 10)?;
-            .map(|mask| {
+            check_netmask(mask, false)?;
-                if !(mask > 0 && mask <= 32) {
+            return Ok((address.to_string(), Some(mask), false));
-                    bail!("IPv4 mask '{}' is out of range (1..32).", mask);
+        } else {
-                }
+            return Ok((address.to_string(), None, false));
-                Ok(mask)
+        }
-            })?;
-        return Ok((address.to_string(), mask.unwrap(), false));
     } else if let Some(caps) = CIDR_V6_REGEX.captures(&cidr) {
         let address = &caps[1];
-        let mask = &caps[2];
+        if let Some(mask) = caps.get(2) {
-        let mask = u8::from_str_radix(mask, 10)
+            let mask = u8::from_str_radix(mask.as_str(), 10)?;
-            .map(|mask| {
+            check_netmask(mask, true)?;
-                if !(mask >= 1 && mask <= 128) {
+            return Ok((address.to_string(), Some(mask), true));
-                    bail!("IPv6 mask '{}' is out of range (1..128).", mask);
+        } else {
-                }
+            return Ok((address.to_string(), None, true));
-                Ok(mask)
+        }
-            })?;
-        return Ok((address.to_string(), mask.unwrap(), true));
     } else {
         bail!("invalid address/mask '{}'", cidr);
     }

src/config/network/parser.rs

@@ -86,20 +86,35 @@ impl <R: BufRead> NetworkParser<R> {
         Ok(())
     }
 
-    fn parse_iface_address(&mut self, interface: &mut Interface) -> Result<(), Error> {
+    fn parse_netmask(&mut self) -> Result<u8, Error> {
-        self.eat(Token::Address)?;
+        self.eat(Token::Netmask)?;
-        let cidr = self.next_text()?;
+        let netmask = self.next_text()?;
 
-        let (_address, _mask, ipv6) = parse_cidr(&cidr)?;
+        let mask = if let Some(mask) = IPV4_MASK_HASH_LOCALNET.get(netmask.as_str())  {
-        if ipv6 {
+            *mask
-            interface.set_cidr_v6(cidr)?;
         } else {
-            interface.set_cidr_v4(cidr)?;
+            match u8::from_str_radix(netmask.as_str(), 10) {
-        }
+                Ok(mask) => mask,
+                Err(err) => {
+                    bail!("unable to parse netmask '{}' - {}", netmask, err);
+                }
+            }
+        };
 
         self.eat(Token::Newline)?;
 
-        Ok(())
+        Ok(mask)
+    }
+
+    fn parse_iface_address(&mut self) -> Result<(String, Option<u8>, bool), Error> {
+        self.eat(Token::Address)?;
+        let cidr = self.next_text()?;
+
+        let (_address, mask, ipv6) = parse_address_or_cidr(&cidr)?;
+
+        self.eat(Token::Newline)?;
+
+        Ok((cidr, mask, ipv6))
     }
 
     fn parse_iface_gateway(&mut self, interface: &mut Interface) -> Result<(), Error> {
@@ -191,6 +206,9 @@ impl <R: BufRead> NetworkParser<R> {
         address_family_v6: bool,
     ) -> Result<(), Error> {
 
+        let mut netmask = None;
+        let mut address_list = Vec::new();
+
         loop {
             match self.peek()? {
                 Token::Attribute => { self.eat(Token::Attribute)?; },
@@ -214,8 +232,15 @@ impl <R: BufRead> NetworkParser<R> {
             }
 
             match self.peek()? {
-                Token::Address => self.parse_iface_address(interface)?,
+                Token::Address => {
+                    let (cidr, mask, is_v6) = self.parse_iface_address()?;
+                    address_list.push((cidr, mask, is_v6));
+                }
                 Token::Gateway => self.parse_iface_gateway(interface)?,
+                Token::Netmask => {
+                    //Note: netmask is deprecated, but we try to do our best
+                    netmask = Some(self.parse_netmask()?);
+                }
                 Token::MTU => {
                     let mtu = self.parse_iface_mtu()?;
                     interface.mtu = Some(mtu);
@@ -255,8 +280,6 @@ impl <R: BufRead> NetworkParser<R> {
                     interface.bond_xmit_hash_policy = Some(policy);
                     self.eat(Token::Newline)?;
                 }
-                Token::Netmask => bail!("netmask is deprecated and no longer supported"),
-
                 _ => { // parse addon attributes
                     let option = self.parse_to_eol()?;
                     if !option.is_empty() {
@@ -270,6 +293,38 @@ impl <R: BufRead> NetworkParser<R> {
             }
         }
 
+        if let Some(netmask) = netmask {
+            if address_list.len() > 1 {
+                bail!("unable to apply netmask to multiple addresses (please use cidr notation)");
+            } else if address_list.len() == 1 {
+                let (mut cidr, mask, is_v6) = address_list.pop().unwrap();
+                if mask.is_some()  {
+                    // address already has a mask  - ignore netmask
+                } else {
+                    check_netmask(netmask, is_v6)?;
+                    cidr.push_str(&format!("/{}", netmask));
+                }
+                if is_v6 {
+                    interface.set_cidr_v6(cidr)?;
+                } else {
+                    interface.set_cidr_v4(cidr)?;
+                }
+            } else {
+                // no address - simply ignore useless netmask
+            }
+        } else {
+            for (cidr, mask, is_v6) in address_list {
+                if mask.is_none() {
+                    bail!("missing netmask in '{}'", cidr);
+                }
+                if is_v6 {
+                    interface.set_cidr_v6(cidr)?;
+                } else {
+                    interface.set_cidr_v4(cidr)?;
+                }
+            }
+        }
+
         Ok(())
     }
 

src/config/sync.rs

@@ -21,7 +21,6 @@ lazy_static! {
     static ref CONFIG: SectionConfig = init();
 }
 
-
 #[api(
     properties: {
         id: {
@@ -30,6 +29,10 @@ lazy_static! {
         store: {
            schema: DATASTORE_SCHEMA,
         },
+        "owner": {
+            type: Authid,
+            optional: true,
+        },
         remote: {
             schema: REMOTE_ID_SCHEMA,
         },
@@ -56,6 +59,8 @@ lazy_static! {
 pub struct SyncJobConfig {
     pub id: String,
     pub store: String,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub owner: Option<Authid>,
     pub remote: String,
     pub remote_store: String,
     #[serde(skip_serializing_if="Option::is_none")]
@@ -66,6 +71,21 @@ pub struct SyncJobConfig {
     pub schedule: Option<String>,
 }
 
+impl From<&SyncJobStatus> for SyncJobConfig {
+    fn from(job_status: &SyncJobStatus) -> Self {
+        Self {
+            id: job_status.id.clone(),
+            store: job_status.store.clone(),
+            owner: job_status.owner.clone(),
+            remote: job_status.remote.clone(),
+            remote_store: job_status.remote_store.clone(),
+            remove_vanished: job_status.remove_vanished.clone(),
+            comment: job_status.comment.clone(),
+            schedule: job_status.schedule.clone(),
+        }
+    }
+}
+
 // FIXME: generate duplicate schemas/structs from one listing?
 #[api(
     properties: {
@@ -75,6 +95,10 @@ pub struct SyncJobConfig {
         store: {
            schema: DATASTORE_SCHEMA,
         },
+        owner: {
+            type: Authid,
+            optional: true,
+        },
         remote: {
             schema: REMOTE_ID_SCHEMA,
         },
@@ -121,6 +145,8 @@ pub struct SyncJobConfig {
 pub struct SyncJobStatus {
     pub id: String,
     pub store: String,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub owner: Option<Authid>,
     pub remote: String,
     pub remote_store: String,
     #[serde(skip_serializing_if="Option::is_none")]

src/config/user.rs

@@ -241,6 +241,14 @@ pub fn save_config(config: &SectionConfigData) -> Result<(), Error> {
     Ok(())
 }
 
+#[cfg(test)]
+pub(crate) fn test_cfg_from_str(raw: &str) -> Result<(SectionConfigData, [u8;32]), Error> {
+    let cfg = init();
+    let parsed = cfg.parse("test_user_cfg", raw)?;
+
+    Ok((parsed, [0;32]))
+}
+
 // shell completion helper
 pub fn complete_userid(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
     match config() {

src/pxar/create.rs

@@ -657,7 +657,12 @@ impl<'a, 'b> Archiver<'a, 'b> {
         let mut remaining = file_size;
         let mut out = encoder.create_file(metadata, file_name, file_size)?;
         while remaining != 0 {
-            let mut got = file.read(&mut self.file_copy_buffer[..])?;
+            let mut got = match file.read(&mut self.file_copy_buffer[..]) {
+                Ok(0) => break,
+                Ok(got) => got,
+                Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue,
+                Err(err) => bail!(err),
+            };
             if got as u64 > remaining {
                 self.report_file_grew_while_reading()?;
                 got = remaining as usize;

src/server.rs

@@ -4,6 +4,47 @@
 //! services. We want async IO, so this is built on top of
 //! tokio/hyper.
 
+use anyhow::{format_err, Error};
+use lazy_static::lazy_static;
+use nix::unistd::Pid;
+
+use proxmox::sys::linux::procfs::PidStat;
+
+use crate::buildcfg;
+
+lazy_static! {
+    static ref PID: i32 = unsafe { libc::getpid() };
+    static ref PSTART: u64 = PidStat::read_from_pid(Pid::from_raw(*PID)).unwrap().starttime;
+}
+
+pub fn pid() -> i32 {
+    *PID
+}
+
+pub fn pstart() -> u64 {
+    *PSTART
+}
+
+pub fn write_pid(pid_fn: &str) -> Result<(), Error> {
+    let pid_str = format!("{}\n", *PID);
+    let opts = proxmox::tools::fs::CreateOptions::new();
+    proxmox::tools::fs::replace_file(pid_fn, pid_str.as_bytes(), opts)
+}
+
+pub fn read_pid(pid_fn: &str) -> Result<i32, Error> {
+    let pid = proxmox::tools::fs::file_get_contents(pid_fn)?;
+    let pid = std::str::from_utf8(&pid)?.trim();
+    pid.parse().map_err(|err| format_err!("could not parse pid - {}", err))
+}
+
+pub fn ctrl_sock_from_pid(pid: i32) -> String {
+    format!("\0{}/control-{}.sock", buildcfg::PROXMOX_BACKUP_RUN_DIR, pid)
+}
+
+pub fn our_ctrl_sock() -> String {
+    ctrl_sock_from_pid(*PID)
+}
+
 mod environment;
 pub use environment::*;
 
@@ -35,5 +76,14 @@ pub mod jobstate;
 mod verify_job;
 pub use verify_job::*;
 
+mod prune_job;
+pub use prune_job::*;
+
+mod gc_job;
+pub use gc_job::*;
+
 mod email_notifications;
 pub use email_notifications::*;
+
+mod report;
+pub use report::*;

src/server/command_socket.rs

@@ -1,17 +1,17 @@
 use anyhow::{bail, format_err, Error};
 
-use futures::*;
+use std::collections::HashMap;
-
-use tokio::net::UnixListener;
-
-use std::path::PathBuf;
-use serde_json::Value;
-use std::sync::Arc;
 use std::os::unix::io::AsRawFd;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use futures::*;
+use tokio::net::UnixListener;
+use serde_json::Value;
 use nix::sys::socket;
 
 /// Listens on a Unix Socket to handle simple command asynchronously
-pub fn create_control_socket<P, F>(path: P, func: F) -> Result<impl Future<Output = ()>, Error>
+fn create_control_socket<P, F>(path: P, func: F) -> Result<impl Future<Output = ()>, Error>
 where
     P: Into<PathBuf>,
     F: Fn(Value) -> Result<Value, Error> + Send + Sync + 'static,
@@ -140,3 +140,76 @@ pub async fn send_command<P>(
             }
         }).await
 }
+
+/// A callback for a specific commando socket.
+pub type CommandoSocketFn = Box<(dyn Fn(Option<&Value>) -> Result<Value, Error> + Send + Sync + 'static)>;
+
+/// Tooling to get a single control command socket where one can register multiple commands
+/// dynamically.
+/// You need to call `spawn()` to make the socket active.
+pub struct CommandoSocket {
+    socket: PathBuf,
+    commands: HashMap<String, CommandoSocketFn>,
+}
+
+impl CommandoSocket {
+    pub fn new<P>(path: P) -> Self
+        where P: Into<PathBuf>,
+    {
+        CommandoSocket {
+            socket: path.into(),
+            commands: HashMap::new(),
+        }
+    }
+
+    /// Spawn the socket and consume self, meaning you cannot register commands anymore after
+    /// calling this.
+    pub fn spawn(self) -> Result<(), Error> {
+        let control_future = create_control_socket(self.socket.to_owned(), move |param| {
+            let param = param
+                .as_object()
+                .ok_or_else(|| format_err!("unable to parse parameters (expected json object)"))?;
+
+            let command = match param.get("command") {
+                Some(Value::String(command)) => command.as_str(),
+                None => bail!("no command"),
+                _ => bail!("unable to parse command"),
+            };
+
+            if !self.commands.contains_key(command) {
+                bail!("got unknown command '{}'", command);
+            }
+
+            match self.commands.get(command) {
+                None => bail!("got unknown command '{}'", command),
+                Some(handler) => {
+                    let args = param.get("args"); //.unwrap_or(&Value::Null);
+                    (handler)(args)
+                },
+            }
+        })?;
+
+        tokio::spawn(control_future);
+
+        Ok(())
+    }
+
+    /// Register a new command with a callback.
+    pub fn register_command<F>(
+        &mut self,
+        command: String,
+        handler: F,
+    ) -> Result<(), Error>
+    where
+        F: Fn(Option<&Value>) -> Result<Value, Error> + Send + Sync + 'static,
+    {
+
+        if self.commands.contains_key(&command) {
+            bail!("command '{}' already exists!", command);
+        }
+
+        self.commands.insert(command, Box::new(handler));
+
+        Ok(())
+    }
+}

src/server/config.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 use std::path::PathBuf;
 use std::time::SystemTime;
 use std::fs::metadata;
-use std::sync::{Mutex, RwLock};
+use std::sync::{Arc, Mutex, RwLock};
 
 use anyhow::{bail, Error, format_err};
 use hyper::Method;
@@ -21,7 +21,7 @@ pub struct ApiConfig {
     env_type: RpcEnvironmentType,
     templates: RwLock<Handlebars<'static>>,
     template_files: RwLock<HashMap<String, (SystemTime, PathBuf)>>,
-    request_log: Option<Mutex<FileLogger>>,
+    request_log: Option<Arc<Mutex<FileLogger>>>,
 }
 
 impl ApiConfig {
@@ -124,7 +124,11 @@ impl ApiConfig {
         }
     }
 
-    pub fn enable_file_log<P>(&mut self, path: P) -> Result<(), Error>
+    pub fn enable_file_log<P>(
+        &mut self,
+        path: P,
+        commando_sock: &mut super::CommandoSocket,
+    ) -> Result<(), Error>
     where
         P: Into<PathBuf>
     {
@@ -142,11 +146,19 @@ impl ApiConfig {
             owned_by_backup: true,
             ..Default::default()
         };
-        self.request_log = Some(Mutex::new(FileLogger::new(&path, logger_options)?));
+        let request_log = Arc::new(Mutex::new(FileLogger::new(&path, logger_options)?));
+        self.request_log = Some(Arc::clone(&request_log));
+
+        commando_sock.register_command("api-access-log-reopen".into(), move |_args| {
+            println!("re-opening log file");
+            request_log.lock().unwrap().reopen()?;
+            Ok(serde_json::Value::Null)
+        })?;
 
         Ok(())
     }
-    pub fn get_file_log(&self) -> Option<&Mutex<FileLogger>> {
+
+    pub fn get_file_log(&self) -> Option<&Arc<Mutex<FileLogger>>> {
         self.request_log.as_ref()
     }
 }

src/server/email_notifications.rs

@@ -4,13 +4,18 @@ use serde_json::json;
 use handlebars::{Handlebars, Helper, Context, RenderError, RenderContext, Output, HelperResult};
 
 use proxmox::tools::email::sendmail;
+use proxmox::api::schema::parse_property_string;
 
 use crate::{
+    config::datastore::DataStoreConfig,
     config::verify::VerificationJobConfig,
     config::sync::SyncJobConfig,
     api2::types::{
-        Userid,
+        APTUpdateInfo,
         GarbageCollectionStatus,
+        Userid,
+        Notify,
+        DatastoreNotify,
     },
     tools::format::HumanByte,
 };
@@ -23,19 +28,24 @@ Index file count:     {{status.index-file-count}}
 
 Removed garbage:      {{human-bytes status.removed-bytes}}
 Removed chunks:       {{status.removed-chunks}}
-Remove bad files:     {{status.removed-bad}}
+Removed bad chunks:   {{status.removed-bad}}
 
-Bad files:            {{status.still-bad}}
+Leftover bad chunks:  {{status.still-bad}}
 Pending removals:     {{human-bytes status.pending-bytes}} (in {{status.pending-chunks}} chunks)
 
 Original Data usage:  {{human-bytes status.index-data-bytes}}
-On Disk usage:        {{human-bytes status.disk-bytes}} ({{relative-percentage status.disk-bytes status.index-data-bytes}})
+On-Disk usage:        {{human-bytes status.disk-bytes}} ({{relative-percentage status.disk-bytes status.index-data-bytes}})
-On Disk chunks:       {{status.disk-chunks}}
+On-Disk chunks:       {{status.disk-chunks}}
 
 Deduplication Factor: {{deduplication-factor}}
 
 Garbage collection successful.
 
+
+Please visit the web interface for futher details:
+
+<https://{{fqdn}}:{{port}}/#DataStore-{{datastore}}>
+
 "###;
 
 
@@ -45,6 +55,11 @@ Datastore: {{datastore}}
 
 Garbage collection failed: {{error}}
 
+
+Please visit the web interface for futher details:
+
+<https://{{fqdn}}:{{port}}/#pbsServerAdministration:tasks>
+
 "###;
 
 const VERIFY_OK_TEMPLATE: &str = r###"
@@ -54,6 +69,11 @@ Datastore: {{job.store}}
 
 Verification successful.
 
+
+Please visit the web interface for futher details:
+
+<https://{{fqdn}}:{{port}}/#DataStore-{{job.store}}>
+
 "###;
 
 const VERIFY_ERR_TEMPLATE: &str = r###"
@@ -64,9 +84,14 @@ Datastore: {{job.store}}
 Verification failed on these snapshots:
 
 {{#each errors}}
-  {{this}}
+  {{this~}}
 {{/each}}
 
+
+Please visit the web interface for futher details:
+
+<https://{{fqdn}}:{{port}}/#pbsServerAdministration:tasks>
+
 "###;
 
 const SYNC_OK_TEMPLATE: &str = r###"
@@ -78,6 +103,11 @@ Remote Store: {{job.remote-store}}
 
 Synchronization successful.
 
+
+Please visit the web interface for futher details:
+
+<https://{{fqdn}}:{{port}}/#DataStore-{{job.store}}>
+
 "###;
 
 const SYNC_ERR_TEMPLATE: &str = r###"
@@ -89,8 +119,26 @@ Remote Store: {{job.remote-store}}
 
 Synchronization failed: {{error}}
 
+
+Please visit the web interface for futher details:
+
+<https://{{fqdn}}:{{port}}/#pbsServerAdministration:tasks>
+
 "###;
 
+const PACKAGE_UPDATES_TEMPLATE: &str = r###"
+Proxmox Backup Server has the following updates available:
+{{#each updates }}
+  {{Package}}: {{OldVersion}} -> {{Version~}}
+{{/each }}
+
+To upgrade visit the web interface:
+
+<https://{{fqdn}}:{{port}}/#pbsServerAdministration:updates>
+
+"###;
+
+
 lazy_static::lazy_static!{
 
     static ref HANDLEBARS: Handlebars<'static> = {
@@ -110,6 +158,8 @@ lazy_static::lazy_static!{
         hb.register_template_string("sync_ok_template", SYNC_OK_TEMPLATE).unwrap();
         hb.register_template_string("sync_err_template", SYNC_ERR_TEMPLATE).unwrap();
 
+        hb.register_template_string("package_update_template", PACKAGE_UPDATES_TEMPLATE).unwrap();
+
         hb
     };
 }
@@ -142,11 +192,28 @@ fn send_job_status_mail(
 
 pub fn send_gc_status(
     email: &str,
+    notify: DatastoreNotify,
     datastore: &str,
     status: &GarbageCollectionStatus,
     result: &Result<(), Error>,
 ) -> Result<(), Error> {
 
+    match notify.gc {
+        None => { /* send notifications by default */ },
+        Some(notify) => {
+            if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
+                return Ok(());
+            }
+        }
+    }
+
+    let (fqdn, port) = get_server_url();
+    let mut data = json!({
+        "datastore": datastore,
+        "fqdn": fqdn,
+        "port": port,
+    });
+
     let text = match result {
         Ok(()) => {
             let deduplication_factor = if status.disk_bytes > 0 {
@@ -155,19 +222,13 @@ pub fn send_gc_status(
                 1.0
             };
 
-            let data = json!({
+            data["status"] = json!(status);
-                "status": status,
+            data["deduplication-factor"] = format!("{:.2}", deduplication_factor).into();
-                "datastore": datastore,
-                "deduplication-factor": format!("{:.2}", deduplication_factor),
-            });
 
             HANDLEBARS.render("gc_ok_template", &data)?
         }
         Err(err) => {
-            let data = json!({
+            data["error"] = err.to_string().into();
-                "error": err.to_string(),
-                "datastore": datastore,
-            });
             HANDLEBARS.render("gc_err_template", &data)?
         }
     };
@@ -190,26 +251,44 @@ pub fn send_gc_status(
 
 pub fn send_verify_status(
     email: &str,
+    notify: DatastoreNotify,
     job: VerificationJobConfig,
     result: &Result<Vec<String>, Error>,
 ) -> Result<(), Error> {
 
+    let (fqdn, port) = get_server_url();
+    let mut data = json!({
+        "job": job,
+        "fqdn": fqdn,
+        "port": port,
+    });
+
+    let mut result_is_ok = false;
 
     let text = match result {
         Ok(errors) if errors.is_empty() => {
-            let data = json!({ "job": job });
+            result_is_ok = true;
             HANDLEBARS.render("verify_ok_template", &data)?
         }
         Ok(errors) => {
-            let data = json!({ "job": job, "errors": errors });
+            data["errors"] = json!(errors);
             HANDLEBARS.render("verify_err_template", &data)?
         }
         Err(_) => {
-            // aboreted job - do not send any email
+            // aborted job - do not send any email
             return Ok(());
         }
     };
 
+    match notify.verify {
+        None => { /* send notifications by default */ },
+        Some(notify) => {
+            if notify == Notify::Never || (result_is_ok && notify == Notify::Error) {
+                return Ok(());
+            }
+        }
+    }
+
     let subject = match result {
         Ok(errors) if errors.is_empty() => format!(
             "Verify Datastore '{}' successful",
@@ -228,17 +307,33 @@ pub fn send_verify_status(
 
 pub fn send_sync_status(
     email: &str,
+    notify: DatastoreNotify,
     job: &SyncJobConfig,
     result: &Result<(), Error>,
 ) -> Result<(), Error> {
 
+    match notify.sync {
+        None => { /* send notifications by default */ },
+        Some(notify) => {
+            if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
+                return Ok(());
+            }
+        }
+    }
+
+    let (fqdn, port) = get_server_url();
+    let mut data = json!({
+        "job": job,
+        "fqdn": fqdn,
+        "port": port,
+    });
+
     let text = match result {
         Ok(()) => {
-            let data = json!({ "job": job });
             HANDLEBARS.render("sync_ok_template", &data)?
         }
         Err(err) => {
-            let data = json!({ "job": job, "error": err.to_string() });
+            data["error"] = err.to_string().into();
             HANDLEBARS.render("sync_err_template", &data)?
         }
     };
@@ -261,17 +356,51 @@ pub fn send_sync_status(
     Ok(())
 }
 
+fn get_server_url() -> (String, usize) {
+
+    // user will surely request that they can change this
+
+    let nodename = proxmox::tools::nodename();
+    let mut fqdn = nodename.to_owned();
+
+    if let Ok(resolv_conf) = crate::api2::node::dns::read_etc_resolv_conf() {
+        if let Some(search) = resolv_conf["search"].as_str() {
+            fqdn.push('.');
+            fqdn.push_str(search);
+        }
+    }
+
+    let port = 8007;
+
+    (fqdn, port)
+}
+
+pub fn send_updates_available(
+    updates: &Vec<&APTUpdateInfo>,
+) -> Result<(), Error> {
+    // update mails always go to the root@pam configured email..
+    if let Some(email) = lookup_user_email(Userid::root_userid()) {
+        let nodename = proxmox::tools::nodename();
+        let subject = format!("New software packages available ({})", nodename);
+
+        let (fqdn, port) = get_server_url();
+
+        let text = HANDLEBARS.render("package_update_template", &json!({
+            "fqdn": fqdn,
+            "port": port,
+            "updates": updates,
+        }))?;
+
+        send_job_status_mail(&email, &subject, &text)?;
+    }
+    Ok(())
+}
+
 /// Lookup users email address
-///
+fn lookup_user_email(userid: &Userid) -> Option<String> {
-/// For "backup@pam", this returns the address from "root@pam".
-pub fn lookup_user_email(userid: &Userid) -> Option<String> {
 
     use crate::config::user::{self, User};
 
-    if userid == Userid::backup_userid() {
-        return lookup_user_email(Userid::root_userid());
-    }
-
     if let Ok(user_config) = user::cached_config() {
         if let Ok(user) = user_config.lookup::<User>("user", userid.as_str()) {
             return user.email.clone();
@@ -281,6 +410,41 @@ pub fn lookup_user_email(userid: &Userid) -> Option<String> {
     None
 }
 
+/// Lookup Datastore notify settings
+pub fn lookup_datastore_notify_settings(
+    store: &str,
+) -> (Option<String>, DatastoreNotify) {
+
+    let mut email = None;
+
+    let notify = DatastoreNotify { gc: None, verify: None, sync: None };
+
+    let (config, _digest) = match crate::config::datastore::config() {
+        Ok(result) => result,
+        Err(_) => return (email, notify),
+    };
+
+    let config: DataStoreConfig = match config.lookup("datastore", store) {
+        Ok(result) => result,
+        Err(_) => return (email, notify),
+    };
+
+    email = match config.notify_user {
+        Some(ref userid) => lookup_user_email(userid),
+        None => lookup_user_email(Userid::root_userid()),
+    };
+
+    let notify_str = config.notify.unwrap_or(String::new());
+
+    if let Ok(value) = parse_property_string(&notify_str, &DatastoreNotify::API_SCHEMA) {
+        if let Ok(notify) = serde_json::from_value(value) {
+            return (email, notify);
+        }
+    }
+
+    (email, notify)
+}
+
 // Handlerbar helper functions
 
 fn handlebars_humam_bytes_helper(

src/server/gc_job.rs

@@ -0,0 +1,63 @@
+use std::sync::Arc;
+use anyhow::Error;
+
+use crate::{
+    server::WorkerTask,
+    api2::types::*,
+    server::jobstate::Job,
+    backup::DataStore,
+};
+
+/// Runs a garbage collection job.
+pub fn do_garbage_collection_job(
+    mut job: Job,
+    datastore: Arc<DataStore>,
+    auth_id: &Authid,
+    schedule: Option<String>,
+    to_stdout: bool,
+) -> Result<String, Error> {
+
+    let store = datastore.name().to_string();
+
+    let (email, notify) = crate::server::lookup_datastore_notify_settings(&store);
+
+    let worker_type = job.jobtype().to_string();
+    let upid_str = WorkerTask::new_thread(
+        &worker_type,
+        Some(store.clone()),
+        auth_id.clone(),
+        to_stdout,
+        move |worker| {
+            job.start(&worker.upid().to_string())?;
+
+            worker.log(format!("starting garbage collection on store {}", store));
+            if let Some(event_str) = schedule {
+                worker.log(format!("task triggered by schedule '{}'", event_str));
+            }
+
+            let result = datastore.garbage_collection(&*worker, worker.upid());
+
+            let status = worker.create_state(&result);
+
+            match job.finish(status) {
+                Err(err) => eprintln!(
+                    "could not finish job state for {}: {}",
+                    job.jobtype().to_string(),
+                    err
+                ),
+                Ok(_) => (),
+            }
+
+            if let Some(email) = email {
+                let gc_status = datastore.last_gc_status();
+                if let Err(err) = crate::server::send_gc_status(&email, notify, &store, &gc_status, &result) {
+                    eprintln!("send gc notification failed: {}", err);
+                }
+            }
+
+            result
+        }
+    )?;
+
+    Ok(upid_str)
+}

src/server/prune_job.rs

@@ -0,0 +1,91 @@
+use anyhow::Error;
+
+use proxmox::try_block;
+
+use crate::{
+    api2::types::*,
+    backup::{compute_prune_info, BackupGroup, DataStore, PruneOptions},
+    server::jobstate::Job,
+    server::WorkerTask,
+    task_log,
+};
+
+pub fn do_prune_job(
+    mut job: Job,
+    prune_options: PruneOptions,
+    store: String,
+    auth_id: &Authid,
+    schedule: Option<String>,
+) -> Result<String, Error> {
+    let datastore = DataStore::lookup_datastore(&store)?;
+
+    let worker_type = job.jobtype().to_string();
+    let upid_str = WorkerTask::new_thread(
+        &worker_type,
+        Some(job.jobname().to_string()),
+        auth_id.clone(),
+        false,
+        move |worker| {
+            job.start(&worker.upid().to_string())?;
+
+            let result = try_block!({
+                task_log!(worker, "Starting datastore prune on store \"{}\"", store);
+
+                if let Some(event_str) = schedule {
+                    task_log!(worker, "task triggered by schedule '{}'", event_str);
+                }
+
+                task_log!(
+                    worker,
+                    "retention options: {}",
+                    prune_options.cli_options_string()
+                );
+
+                let base_path = datastore.base_path();
+
+                let groups = BackupGroup::list_groups(&base_path)?;
+                for group in groups {
+                    let list = group.list_backups(&base_path)?;
+                    let mut prune_info = compute_prune_info(list, &prune_options)?;
+                    prune_info.reverse(); // delete older snapshots first
+
+                    task_log!(
+                        worker,
+                        "Starting prune on store \"{}\" group \"{}/{}\"",
+                        store,
+                        group.backup_type(),
+                        group.backup_id()
+                    );
+
+                    for (info, keep) in prune_info {
+                        task_log!(
+                            worker,
+                            "{} {}/{}/{}",
+                            if keep { "keep" } else { "remove" },
+                            group.backup_type(),
+                            group.backup_id(),
+                            info.backup_dir.backup_time_string()
+                        );
+                        if !keep {
+                            datastore.remove_backup_dir(&info.backup_dir, true)?;
+                        }
+                    }
+                }
+                Ok(())
+            });
+
+            let status = worker.create_state(&result);
+
+            if let Err(err) = job.finish(status) {
+                eprintln!(
+                    "could not finish job state for {}: {}",
+                    job.jobtype().to_string(),
+                    err
+                );
+            }
+
+            result
+        },
+    )?;
+    Ok(upid_str)
+}

src/server/report.rs

@@ -0,0 +1,91 @@
+use std::path::Path;
+use std::process::Command;
+
+use crate::config::datastore;
+
+fn files() -> Vec<&'static str> {
+    vec![
+        "/etc/hostname",
+        "/etc/hosts",
+        "/etc/network/interfaces",
+        "/etc/proxmox-backup/datastore.cfg",
+        "/etc/proxmox-backup/user.cfg",
+        "/etc/proxmox-backup/acl.cfg",
+        "/etc/proxmox-backup/remote.cfg",
+        "/etc/proxmox-backup/sync.cfg",
+        "/etc/proxmox-backup/verification.cfg",
+    ]
+}
+
+fn commands() -> Vec<(&'static str, Vec<&'static str>)> {
+    vec![
+    //  ("<command>", vec![<arg [, arg]>])
+        ("df", vec!["-h"]),
+        ("lsblk", vec!["--ascii"]),
+        ("zpool", vec!["status"]),
+        ("zfs", vec!["list"]),
+        ("proxmox-backup-manager", vec!["subscription", "get"]),
+    ]
+}
+
+    // (<description>, <function to call>)
+fn function_calls() -> Vec<(&'static str, fn() -> String)> {
+    vec![
+        ("Datastores", || {
+            let config = match datastore::config() {
+                Ok((config, _digest)) => config,
+                _ => return String::from("could not read datastore config"),
+            };
+
+            let mut list = Vec::new();
+            for (store, _) in &config.sections {
+                list.push(store.as_str());
+            }
+            list.join(", ")
+        })
+    ]
+}
+
+pub fn generate_report() -> String {
+    use proxmox::tools::fs::file_read_optional_string;
+
+    let file_contents = files()
+        .iter()
+        .map(|file_name| {
+            let content = match file_read_optional_string(Path::new(file_name)) {
+                Ok(Some(content)) => content,
+                Ok(None) => String::from("# file does not exists"),
+                Err(err) => err.to_string(),
+            };
+            format!("# cat '{}'\n{}", file_name, content)
+        })
+        .collect::<Vec<String>>()
+        .join("\n\n");
+
+    let command_outputs = commands()
+        .iter()
+        .map(|(command, args)| {
+            let output = Command::new(command)
+                .env("PROXMOX_OUTPUT_NO_BORDER", "1")
+                .args(args)
+                .output();
+            let output = match output {
+                Ok(output) => String::from_utf8_lossy(&output.stdout).to_string(),
+                Err(err) => err.to_string(),
+            };
+            format!("# `{} {}`\n{}", command, args.join(" "), output)
+        })
+        .collect::<Vec<String>>()
+        .join("\n\n");
+
+    let function_outputs = function_calls()
+        .iter()
+        .map(|(desc, function)| format!("# {}\n{}", desc, function()))
+        .collect::<Vec<String>>()
+        .join("\n\n");
+
+    format!(
+        "= FILES =\n\n{}\n= COMMANDS =\n\n{}\n= FUNCTIONS =\n\n{}\n",
+        file_contents, command_outputs, function_outputs
+    )
+}

src/server/rest.rs

@@ -17,6 +17,7 @@ use lazy_static::lazy_static;
 use serde_json::{json, Value};
 use tokio::fs::File;
 use tokio::time::Instant;
+use percent_encoding::percent_decode_str;
 use url::form_urlencoded;
 use regex::Regex;
 
@@ -111,7 +112,7 @@ pub struct ApiService {
 }
 
 fn log_response(
-    logfile: Option<&Mutex<FileLogger>>,
+    logfile: Option<&Arc<Mutex<FileLogger>>>,
     peer: &std::net::SocketAddr,
     method: hyper::Method,
     path_query: &str,
@@ -163,6 +164,15 @@ fn log_response(
             ));
     }
 }
+pub fn auth_logger() -> Result<FileLogger, Error> {
+    let logger_options = tools::FileLogOptions {
+        append: true,
+        prefix_time: true,
+        owned_by_backup: true,
+        ..Default::default()
+    };
+    FileLogger::new(crate::buildcfg::API_AUTH_LOG_FN, logger_options)
+}
 
 fn get_proxied_peer(headers: &HeaderMap) -> Option<std::net::SocketAddr> {
     lazy_static! {
@@ -552,7 +562,7 @@ enum AuthData {
 }
 
 fn extract_auth_data(headers: &http::HeaderMap) -> Option<AuthData> {
-    if let Some(raw_cookie) = headers.get("COOKIE") {
+    if let Some(raw_cookie) = headers.get(header::COOKIE) {
         if let Ok(cookie) = raw_cookie.to_str() {
             if let Some(ticket) = tools::extract_cookie(cookie, "PBSAuthCookie") {
                 let csrf_token = match headers.get("CSRFPreventionToken").map(|v| v.to_str()) {
@@ -567,8 +577,14 @@ fn extract_auth_data(headers: &http::HeaderMap) -> Option<AuthData> {
         }
     }
 
-    match headers.get("AUTHORIZATION").map(|v| v.to_str()) {
+    match headers.get(header::AUTHORIZATION).map(|v| v.to_str()) {
-        Some(Ok(v)) => Some(AuthData::ApiToken(v.to_owned())),
+        Some(Ok(v)) => {
+            if v.starts_with("PBSAPIToken ") || v.starts_with("PBSAPIToken=") {
+                Some(AuthData::ApiToken(v["PBSAPIToken ".len()..].to_owned()))
+            } else {
+                None
+            }
+        },
         _ => None,
     }
 }
@@ -609,6 +625,10 @@ fn check_auth(
 
             let tokensecret = parts.next()
                 .ok_or_else(|| format_err!("failed to split API token header"))?;
+            let tokensecret = percent_decode_str(tokensecret)
+                .decode_utf8()
+                .map_err(|_| format_err!("failed to decode API token header"))?;
+
             crate::config::token_shadow::verify_secret(&tokenid, &tokensecret)?;
 
             Ok(tokenid)
@@ -676,6 +696,10 @@ async fn handle_request(
                 match auth_result {
                     Ok(authid) => rpcenv.set_auth_id(Some(authid.to_string())),
                     Err(err) => {
+                        let peer = peer.ip();
+                        auth_logger()?
+                            .log(format!("authentication failure; rhost={} msg={}", peer, err));
+
                         // always delay unauthorized calls by 3 seconds (from start of request)
                         let err = http_err!(UNAUTHORIZED, "authentication failed - {}", err);
                         tokio::time::delay_until(Instant::from_std(delay_unauth_time)).await;

src/server/verify_job.rs

@@ -48,13 +48,15 @@ pub fn do_verification_job(
         }
     };
 
-    let email = crate::server::lookup_user_email(auth_id.user());
+    let (email, notify) = crate::server::lookup_datastore_notify_settings(&verification_job.store);
 
-    let job_id = job.jobname().to_string();
+    let job_id = format!("{}:{}",
+                         &verification_job.store,
+                         job.jobname());
     let worker_type = job.jobtype().to_string();
     let upid_str = WorkerTask::new_thread(
         &worker_type,
-        Some(job.jobname().to_string()),
+        Some(job_id.clone()),
         auth_id.clone(),
         false,
         move |worker| {
@@ -65,7 +67,7 @@ pub fn do_verification_job(
                 task_log!(worker,"task triggered by schedule '{}'", event_str);
             }
 
-            let result = verify_all_backups(datastore, worker.clone(), worker.upid(), Some(&filter));
+            let result = verify_all_backups(datastore, worker.clone(), worker.upid(), None, Some(&filter));
             let job_result = match result {
                 Ok(ref errors) if errors.is_empty() => Ok(()),
                 Ok(_) => Err(format_err!("verification failed - please check the log for details")),
@@ -84,7 +86,7 @@ pub fn do_verification_job(
             }
 
             if let Some(email) = email {
-                if let Err(err) = crate::server::send_verify_status(&email, verification_job, &result) {
+                if let Err(err) = crate::server::send_verify_status(&email, notify, verification_job, &result) {
                     eprintln!("send verify notification failed: {}", err);
                 }
             }

src/server/worker_task.rs

@@ -8,7 +8,6 @@ use std::sync::{Arc, Mutex};
 use anyhow::{bail, format_err, Error};
 use futures::*;
 use lazy_static::lazy_static;
-use nix::unistd::Pid;
 use serde_json::{json, Value};
 use serde::{Serialize, Deserialize};
 use tokio::sync::oneshot;
@@ -19,34 +18,33 @@ use proxmox::tools::fs::{create_path, open_file_locked, replace_file, CreateOpti
 
 use super::UPID;
 
+use crate::buildcfg;
+use crate::server;
 use crate::tools::logrotate::{LogRotate, LogRotateFiles};
 use crate::tools::{FileLogger, FileLogOptions};
-use crate::api2::types::Authid;
+use crate::api2::types::{Authid, TaskStateType};
 
-macro_rules! PROXMOX_BACKUP_VAR_RUN_DIR_M { () => ("/run/proxmox-backup") }
+macro_rules! taskdir {
-macro_rules! PROXMOX_BACKUP_LOG_DIR_M { () => ("/var/log/proxmox-backup") }
+    ($subdir:expr) => (concat!(PROXMOX_BACKUP_LOG_DIR_M!(), "/tasks", $subdir))
-macro_rules! PROXMOX_BACKUP_TASK_DIR_M { () => (concat!( PROXMOX_BACKUP_LOG_DIR_M!(), "/tasks")) }
+}
-
+pub const PROXMOX_BACKUP_TASK_DIR: &str = taskdir!("/");
-pub const PROXMOX_BACKUP_VAR_RUN_DIR: &str = PROXMOX_BACKUP_VAR_RUN_DIR_M!();
+pub const PROXMOX_BACKUP_TASK_LOCK_FN: &str = taskdir!("/.active.lock");
-pub const PROXMOX_BACKUP_LOG_DIR: &str = PROXMOX_BACKUP_LOG_DIR_M!();
+pub const PROXMOX_BACKUP_ACTIVE_TASK_FN: &str = taskdir!("/active");
-pub const PROXMOX_BACKUP_TASK_DIR: &str = PROXMOX_BACKUP_TASK_DIR_M!();
+pub const PROXMOX_BACKUP_INDEX_TASK_FN: &str = taskdir!("/index");
-pub const PROXMOX_BACKUP_TASK_LOCK_FN: &str = concat!(PROXMOX_BACKUP_TASK_DIR_M!(), "/.active.lock");
+pub const PROXMOX_BACKUP_ARCHIVE_TASK_FN: &str = taskdir!("/archive");
-pub const PROXMOX_BACKUP_ACTIVE_TASK_FN: &str = concat!(PROXMOX_BACKUP_TASK_DIR_M!(), "/active");
-pub const PROXMOX_BACKUP_INDEX_TASK_FN: &str = concat!(PROXMOX_BACKUP_TASK_DIR_M!(), "/index");
-pub const PROXMOX_BACKUP_ARCHIVE_TASK_FN: &str = concat!(PROXMOX_BACKUP_TASK_DIR_M!(), "/archive");
 
 lazy_static! {
     static ref WORKER_TASK_LIST: Mutex<HashMap<usize, Arc<WorkerTask>>> = Mutex::new(HashMap::new());
+}
 
-    static ref MY_PID: i32 = unsafe { libc::getpid() };
+/// checks if the task UPID refers to a worker from this process
-    static ref MY_PID_PSTART: u64 = procfs::PidStat::read_from_pid(Pid::from_raw(*MY_PID))
+fn is_local_worker(upid: &UPID) -> bool {
-        .unwrap()
+    upid.pid == server::pid() && upid.pstart == server::pstart()
-        .starttime;
 }
 
 /// Test if the task is still running
 pub async fn worker_is_active(upid: &UPID) -> Result<bool, Error> {
-    if (upid.pid == *MY_PID) && (upid.pstart == *MY_PID_PSTART) {
+    if is_local_worker(upid) {
         return Ok(WORKER_TASK_LIST.lock().unwrap().contains_key(&upid.task_id));
     }
 
@@ -54,15 +52,14 @@ pub async fn worker_is_active(upid: &UPID) -> Result<bool, Error> {
         return Ok(false);
     }
 
-    let socketname = format!(
+    let sock = server::ctrl_sock_from_pid(upid.pid);
-        "\0{}/proxmox-task-control-{}.sock", PROXMOX_BACKUP_VAR_RUN_DIR, upid.pid);
-
     let cmd = json!({
-        "command": "status",
+        "command": "worker-task-status",
-        "upid": upid.to_string(),
+        "args": {
+            "upid": upid.to_string(),
+        },
     });
-
+    let status = super::send_command(sock, cmd).await?;
-    let status = super::send_command(socketname, cmd).await?;
 
     if let Some(active) = status.as_bool() {
         Ok(active)
@@ -73,69 +70,48 @@ pub async fn worker_is_active(upid: &UPID) -> Result<bool, Error> {
 
 /// Test if the task is still running (fast but inaccurate implementation)
 ///
-/// If the task is spanned from a different process, we simply return if
+/// If the task is spawned from a different process, we simply return if
 /// that process is still running. This information is good enough to detect
 /// stale tasks...
 pub fn worker_is_active_local(upid: &UPID) -> bool {
-    if (upid.pid == *MY_PID) && (upid.pstart == *MY_PID_PSTART) {
+    if is_local_worker(upid) {
         WORKER_TASK_LIST.lock().unwrap().contains_key(&upid.task_id)
     } else {
         procfs::check_process_running_pstart(upid.pid, upid.pstart).is_some()
     }
 }
 
-pub fn create_task_control_socket() -> Result<(), Error> {
+pub fn register_task_control_commands(
-
+    commando_sock: &mut super::CommandoSocket,
-    let socketname = format!(
+) -> Result<(), Error> {
-        "\0{}/proxmox-task-control-{}.sock", PROXMOX_BACKUP_VAR_RUN_DIR, *MY_PID);
+    fn get_upid(args: Option<&Value>) -> Result<UPID, Error> {
-
+        let args = if let Some(args) = args { args } else { bail!("missing args") };
-    let control_future = super::create_control_socket(socketname, |param| {
+        let upid = match args.get("upid") {
-        let param = param
+            Some(Value::String(upid)) => upid.parse::<UPID>()?,
-            .as_object()
+            None => bail!("no upid in args"),
-            .ok_or_else(|| format_err!("unable to parse parameters (expected json object)"))?;
+            _ => bail!("unable to parse upid"),
-        if param.keys().count() != 2 { bail!("wrong number of parameters"); }
+        };
-
+        if !is_local_worker(&upid) {
-        let command = param["command"]
-            .as_str()
-            .ok_or_else(|| format_err!("unable to parse parameters (missing command)"))?;
-
-        // we have only two commands for now
-        if !(command == "abort-task" || command == "status") {
-            bail!("got unknown command '{}'", command);
-        }
-
-        let upid_str = param["upid"]
-            .as_str()
-            .ok_or_else(|| format_err!("unable to parse parameters (missing upid)"))?;
-
-        let upid = upid_str.parse::<UPID>()?;
-
-        if !(upid.pid == *MY_PID && upid.pstart == *MY_PID_PSTART) {
             bail!("upid does not belong to this process");
         }
+        Ok(upid)
+    }
 
-        let hash = WORKER_TASK_LIST.lock().unwrap();
+    commando_sock.register_command("worker-task-abort".into(), move |args| {
+        let upid = get_upid(args)?;
 
-        match command {
+        if let Some(ref worker) = WORKER_TASK_LIST.lock().unwrap().get(&upid.task_id) {
-            "abort-task" => {
+            worker.request_abort();
-                if let Some(ref worker) = hash.get(&upid.task_id) {
-                    worker.request_abort();
-                } else {
-                    // assume task is already stopped
-                }
-                Ok(Value::Null)
-            }
-            "status" => {
-                let active = hash.contains_key(&upid.task_id);
-                Ok(active.into())
-            }
-            _ => {
-                bail!("got unknown command '{}'", command);
-            }
         }
+        Ok(Value::Null)
     })?;
+    commando_sock.register_command("worker-task-status".into(), move |args| {
+        let upid = get_upid(args)?;
 
-    tokio::spawn(control_future);
+        let active = WORKER_TASK_LIST.lock().unwrap().contains_key(&upid.task_id);
+
+        Ok(active.into())
+    })?;
 
     Ok(())
 }
@@ -150,17 +126,14 @@ pub fn abort_worker_async(upid: UPID) {
 
 pub async fn abort_worker(upid: UPID) -> Result<(), Error> {
 
-    let target_pid = upid.pid;
+    let sock = server::ctrl_sock_from_pid(upid.pid);
-
-    let socketname = format!(
-        "\0{}/proxmox-task-control-{}.sock", PROXMOX_BACKUP_VAR_RUN_DIR, target_pid);
-
     let cmd = json!({
-        "command": "abort-task",
+        "command": "worker-task-abort",
-        "upid": upid.to_string(),
+        "args": {
+            "upid": upid.to_string(),
+        },
     });
-
+    super::send_command(sock, cmd).map_ok(|_| ()).await
-    super::send_command(socketname, cmd).map_ok(|_| ()).await
 }
 
 fn parse_worker_status_line(line: &str) -> Result<(String, UPID, Option<TaskState>), Error> {
@@ -189,9 +162,9 @@ pub fn create_task_log_dirs() -> Result<(), Error> {
             .owner(backup_user.uid)
             .group(backup_user.gid);
 
-        create_path(PROXMOX_BACKUP_LOG_DIR, None, Some(opts.clone()))?;
+        create_path(buildcfg::PROXMOX_BACKUP_LOG_DIR, None, Some(opts.clone()))?;
         create_path(PROXMOX_BACKUP_TASK_DIR, None, Some(opts.clone()))?;
-        create_path(PROXMOX_BACKUP_VAR_RUN_DIR, None, Some(opts))?;
+        create_path(buildcfg::PROXMOX_BACKUP_RUN_DIR, None, Some(opts))?;
         Ok(())
     }).map_err(|err: Error| format_err!("unable to create task log dir - {}", err))?;
 
@@ -273,6 +246,15 @@ impl TaskState {
         }
     }
 
+    pub fn tasktype(&self) -> TaskStateType {
+        match self {
+            TaskState::OK { .. } => TaskStateType::OK,
+            TaskState::Unknown { .. } => TaskStateType::Unknown,
+            TaskState::Error { .. } => TaskStateType::Error,
+            TaskState::Warning { .. } => TaskStateType::Warning,
+        }
+    }
+
     fn result_text(&self) -> String {
         match self {
             TaskState::Error { message, .. } => format!("TASK ERROR: {}", message),
@@ -602,18 +584,9 @@ struct WorkerTaskData {
     pub abort_listeners: Vec<oneshot::Sender<()>>,
 }
 
-impl Drop for WorkerTask {
-
-    fn drop(&mut self) {
-        println!("unregister worker");
-    }
-}
-
 impl WorkerTask {
 
     pub fn new(worker_type: &str, worker_id: Option<String>, auth_id: Authid, to_stdout: bool) -> Result<Arc<Self>, Error> {
-        println!("register worker");
-
         let upid = UPID::new(worker_type, worker_id, auth_id)?;
         let task_id = upid.task_id;
 
@@ -692,8 +665,6 @@ impl WorkerTask {
     ) -> Result<String, Error>
         where F: Send + UnwindSafe + 'static + FnOnce(Arc<WorkerTask>) -> Result<(), Error>
     {
-        println!("register worker thread");
-
         let worker = WorkerTask::new(worker_type, worker_id, auth_id, to_stdout)?;
         let upid_str = worker.upid.to_string();
 

src/tools.rs

@@ -19,6 +19,7 @@ use proxmox::tools::vec;
 pub use proxmox::tools::fd::Fd;
 
 pub mod acl;
+pub mod apt;
 pub mod async_io;
 pub mod borrow;
 pub mod cert;
@@ -462,7 +463,7 @@ pub fn run_command(
    let output = command.output()
         .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
 
-    let output = crate::tools::command_output_as_string(output, exit_code_check)
+    let output = command_output_as_string(output, exit_code_check)
         .map_err(|err| format_err!("command {:?} failed - {}", command, err))?;
 
     Ok(output)

src/tools/apt.rs

@@ -0,0 +1,361 @@
+use std::collections::HashSet;
+use std::collections::HashMap;
+
+use anyhow::{Error, bail, format_err};
+use apt_pkg_native::Cache;
+
+use proxmox::const_regex;
+use proxmox::tools::fs::{file_read_optional_string, replace_file, CreateOptions};
+
+use crate::api2::types::APTUpdateInfo;
+
+const APT_PKG_STATE_FN: &str = "/var/lib/proxmox-backup/pkg-state.json";
+
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+/// Some information we cache about the package (update) state, like what pending update version
+/// we already notfied an user about
+pub struct PkgState {
+    /// simple map from package name to most recently notified (emailed) version
+    pub notified: Option<HashMap<String, String>>,
+    /// A list of pending updates
+    pub package_status: Vec<APTUpdateInfo>,
+}
+
+pub fn write_pkg_cache(state: &PkgState) -> Result<(), Error> {
+    let serialized_state = serde_json::to_string(state)?;
+
+    replace_file(APT_PKG_STATE_FN, &serialized_state.as_bytes(), CreateOptions::new())
+        .map_err(|err| format_err!("Error writing package cache - {}", err))?;
+    Ok(())
+}
+
+pub fn read_pkg_state() -> Result<Option<PkgState>, Error> {
+    let serialized_state = match file_read_optional_string(&APT_PKG_STATE_FN) {
+        Ok(Some(raw)) => raw,
+        Ok(None) => return Ok(None),
+        Err(err) => bail!("could not read cached package state file - {}", err),
+    };
+
+    serde_json::from_str(&serialized_state)
+        .map(|s| Some(s))
+        .map_err(|err| format_err!("could not parse cached package status - {}", err))
+}
+
+pub fn pkg_cache_expired () -> Result<bool, Error> {
+    if let Ok(pbs_cache) = std::fs::metadata(APT_PKG_STATE_FN) {
+        let apt_pkgcache = std::fs::metadata("/var/cache/apt/pkgcache.bin")?;
+        let dpkg_status = std::fs::metadata("/var/lib/dpkg/status")?;
+
+        let mtime = pbs_cache.modified()?;
+
+        if apt_pkgcache.modified()? <= mtime && dpkg_status.modified()? <= mtime {
+            return Ok(false);
+        }
+    }
+    Ok(true)
+}
+
+pub fn update_cache() -> Result<PkgState, Error> {
+        // update our cache
+        let all_upgradeable = list_installed_apt_packages(|data| {
+            data.candidate_version == data.active_version &&
+            data.installed_version != Some(data.candidate_version)
+        }, None);
+
+        let cache = match read_pkg_state() {
+            Ok(Some(mut cache)) => {
+                cache.package_status = all_upgradeable;
+                cache
+            },
+            _ => PkgState {
+                notified: None,
+                package_status: all_upgradeable,
+            },
+        };
+        write_pkg_cache(&cache)?;
+        Ok(cache)
+}
+
+
+const_regex! {
+    VERSION_EPOCH_REGEX = r"^\d+:";
+    FILENAME_EXTRACT_REGEX = r"^.*/.*?_(.*)_Packages$";
+}
+
+// FIXME: once the 'changelog' API call switches over to 'apt-get changelog' only,
+// consider removing this function entirely, as it's value is never used anywhere
+// then (widget-toolkit doesn't use the value either)
+fn get_changelog_url(
+    package: &str,
+    filename: &str,
+    version: &str,
+    origin: &str,
+    component: &str,
+) -> Result<String, Error> {
+    if origin == "" {
+        bail!("no origin available for package {}", package);
+    }
+
+    if origin == "Debian" {
+        let mut command = std::process::Command::new("apt-get");
+        command.arg("changelog");
+        command.arg("--print-uris");
+        command.arg(package);
+        let output = crate::tools::run_command(command, None)?; // format: 'http://foo/bar' package.changelog
+        let output = match output.splitn(2, ' ').next() {
+            Some(output) => {
+                if output.len() < 2 {
+                    bail!("invalid output (URI part too short) from 'apt-get changelog --print-uris': {}", output)
+                }
+                output[1..output.len()-1].to_owned()
+            },
+            None => bail!("invalid output from 'apt-get changelog --print-uris': {}", output)
+        };
+        return Ok(output);
+    } else if origin == "Proxmox" {
+        // FIXME: Use above call to 'apt changelog <pkg> --print-uris' as well.
+        // Currently not possible as our packages do not have a URI set in their Release file.
+        let version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(version, "");
+
+        let base = match (FILENAME_EXTRACT_REGEX.regex_obj)().captures(filename) {
+            Some(captures) => {
+                let base_capture = captures.get(1);
+                match base_capture {
+                    Some(base_underscore) => base_underscore.as_str().replace("_", "/"),
+                    None => bail!("incompatible filename, cannot find regex group")
+                }
+            },
+            None => bail!("incompatible filename, doesn't match regex")
+        };
+
+        return Ok(format!("http://download.proxmox.com/{}/{}_{}.changelog",
+                          base, package, version));
+    }
+
+    bail!("unknown origin ({}) or component ({})", origin, component)
+}
+
+pub struct FilterData<'a> {
+    // this is version info returned by APT
+    pub installed_version: Option<&'a str>,
+    pub candidate_version: &'a str,
+
+    // this is the version info the filter is supposed to check
+    pub active_version: &'a str,
+}
+
+enum PackagePreSelect {
+    OnlyInstalled,
+    OnlyNew,
+    All,
+}
+
+pub fn list_installed_apt_packages<F: Fn(FilterData) -> bool>(
+    filter: F,
+    only_versions_for: Option<&str>,
+) -> Vec<APTUpdateInfo> {
+
+    let mut ret = Vec::new();
+    let mut depends = HashSet::new();
+
+    // note: this is not an 'apt update', it just re-reads the cache from disk
+    let mut cache = Cache::get_singleton();
+    cache.reload();
+
+    let mut cache_iter = match only_versions_for {
+        Some(name) => cache.find_by_name(name),
+        None => cache.iter()
+    };
+
+    loop {
+
+        match cache_iter.next() {
+            Some(view) => {
+                let di = if only_versions_for.is_some() {
+                    query_detailed_info(
+                        PackagePreSelect::All,
+                        &filter,
+                        view,
+                        None
+                    )
+                } else {
+                    query_detailed_info(
+                        PackagePreSelect::OnlyInstalled,
+                        &filter,
+                        view,
+                        Some(&mut depends)
+                    )
+                };
+                if let Some(info) = di {
+                    ret.push(info);
+                }
+
+                if only_versions_for.is_some() {
+                    break;
+                }
+            },
+            None => {
+                drop(cache_iter);
+                // also loop through missing dependencies, as they would be installed
+                for pkg in depends.iter() {
+                    let mut iter = cache.find_by_name(&pkg);
+                    let view = match iter.next() {
+                        Some(view) => view,
+                        None => continue // package not found, ignore
+                    };
+
+                    let di = query_detailed_info(
+                        PackagePreSelect::OnlyNew,
+                        &filter,
+                        view,
+                        None
+                    );
+                    if let Some(info) = di {
+                        ret.push(info);
+                    }
+                }
+                break;
+            }
+        }
+    }
+
+    return ret;
+}
+
+fn query_detailed_info<'a, F, V>(
+    pre_select: PackagePreSelect,
+    filter: F,
+    view: V,
+    depends: Option<&mut HashSet<String>>,
+) -> Option<APTUpdateInfo>
+where
+    F: Fn(FilterData) -> bool,
+    V: std::ops::Deref<Target = apt_pkg_native::sane::PkgView<'a>>
+{
+    let current_version = view.current_version();
+    let candidate_version = view.candidate_version();
+
+    let (current_version, candidate_version) = match pre_select {
+        PackagePreSelect::OnlyInstalled => match (current_version, candidate_version) {
+            (Some(cur), Some(can)) => (Some(cur), can), // package installed and there is an update
+            (Some(cur), None) => (Some(cur.clone()), cur), // package installed and up-to-date
+            (None, Some(_)) => return None, // package could be installed
+            (None, None) => return None, // broken
+        },
+        PackagePreSelect::OnlyNew => match (current_version, candidate_version) {
+            (Some(_), Some(_)) => return None,
+            (Some(_), None) => return None,
+            (None, Some(can)) => (None, can),
+            (None, None) => return None,
+        },
+        PackagePreSelect::All => match (current_version, candidate_version) {
+            (Some(cur), Some(can)) => (Some(cur), can),
+            (Some(cur), None) => (Some(cur.clone()), cur),
+            (None, Some(can)) => (None, can),
+            (None, None) => return None,
+        },
+    };
+
+    // get additional information via nested APT 'iterators'
+    let mut view_iter = view.versions();
+    while let Some(ver) = view_iter.next() {
+
+        let package = view.name();
+        let version = ver.version();
+        let mut origin_res = "unknown".to_owned();
+        let mut section_res = "unknown".to_owned();
+        let mut priority_res = "unknown".to_owned();
+        let mut change_log_url = "".to_owned();
+        let mut short_desc = package.clone();
+        let mut long_desc = "".to_owned();
+
+        let fd = FilterData {
+            installed_version: current_version.as_deref(),
+            candidate_version: &candidate_version,
+            active_version: &version,
+        };
+
+        if filter(fd) {
+            if let Some(section) = ver.section() {
+                section_res = section;
+            }
+
+            if let Some(prio) = ver.priority_type() {
+                priority_res = prio;
+            }
+
+            // assume every package has only one origin file (not
+            // origin, but origin *file*, for some reason those seem to
+            // be different concepts in APT)
+            let mut origin_iter = ver.origin_iter();
+            let origin = origin_iter.next();
+            if let Some(origin) = origin {
+
+                if let Some(sd) = origin.short_desc() {
+                    short_desc = sd;
+                }
+
+                if let Some(ld) = origin.long_desc() {
+                    long_desc = ld;
+                }
+
+                // the package files appear in priority order, meaning
+                // the one for the candidate version is first - this is fine
+                // however, as the source package should be the same for all
+                // versions anyway
+                let mut pkg_iter = origin.file();
+                let pkg_file = pkg_iter.next();
+                if let Some(pkg_file) = pkg_file {
+                    if let Some(origin_name) = pkg_file.origin() {
+                        origin_res = origin_name;
+                    }
+
+                    let filename = pkg_file.file_name();
+                    let component = pkg_file.component();
+
+                    // build changelog URL from gathered information
+                    // ignore errors, use empty changelog instead
+                    let url = get_changelog_url(&package, &filename,
+                        &version, &origin_res, &component);
+                    if let Ok(url) = url {
+                        change_log_url = url;
+                    }
+                }
+            }
+
+            if let Some(depends) = depends {
+                let mut dep_iter = ver.dep_iter();
+                loop {
+                    let dep = match dep_iter.next() {
+                        Some(dep) if dep.dep_type() != "Depends" => continue,
+                        Some(dep) => dep,
+                        None => break
+                    };
+
+                    let dep_pkg = dep.target_pkg();
+                    let name = dep_pkg.name();
+
+                    depends.insert(name);
+                }
+            }
+
+            return Some(APTUpdateInfo {
+                package,
+                title: short_desc,
+                arch: view.arch(),
+                description: long_desc,
+                change_log_url,
+                origin: origin_res,
+                version: candidate_version.clone(),
+                old_version: match current_version {
+                    Some(vers) => vers,
+                    None => "".to_owned()
+                },
+                priority: priority_res,
+                section: section_res,
+            });
+        }
+    }
+
+    return None;
+}

src/tools/daemon.rs

@@ -12,6 +12,7 @@ use std::task::{Context, Poll};
 use std::path::PathBuf;
 
 use anyhow::{bail, format_err, Error};
+use futures::future::{self, Either};
 
 use proxmox::tools::io::{ReadExt, WriteExt};
 
@@ -262,7 +263,7 @@ pub async fn create_daemon<F, S>(
 ) -> Result<(), Error>
 where
     F: FnOnce(tokio::net::TcpListener, NotifyReady) -> Result<S, Error>,
-    S: Future<Output = ()>,
+    S: Future<Output = ()> + Unpin,
 {
     let mut reloader = Reloader::new()?;
 
@@ -271,11 +272,19 @@ where
         move || async move { Ok(tokio::net::TcpListener::bind(&address).await?) },
     ).await?;
 
-    create_service(listener, NotifyReady)?.await;
+    let server_future = create_service(listener, NotifyReady)?;
+    let shutdown_future = server::shutdown_future();
+
+    let finish_future = match future::select(server_future, shutdown_future).await {
+        Either::Left((_, _)) => {
+            crate::tools::request_shutdown(); // make sure we are in shutdown mode
+            None
+        }
+        Either::Right((_, server_future)) => Some(server_future),
+    };
 
     let mut reloader = Some(reloader);
 
-    crate::tools::request_shutdown(); // make sure we are in shutdown mode
     if server::is_reload_request() {
         log::info!("daemon reload...");
         if let Err(e) = systemd_notify(SystemdNotify::Reloading) {
@@ -288,6 +297,11 @@ where
     } else {
         log::info!("daemon shutting down...");
     }
+
+    if let Some(future) = finish_future {
+        future.await;
+    }
+    log::info!("daemon shut down...");
     Ok(())
 }
 

src/tools/file_logger.rs

@@ -47,6 +47,7 @@ pub struct FileLogOptions {
 #[derive(Debug)]
 pub struct FileLogger {
     file: std::fs::File,
+    file_name: std::path::PathBuf,
     options: FileLogOptions,
 }
 
@@ -63,6 +64,23 @@ impl FileLogger {
         file_name: P,
         options: FileLogOptions,
     ) -> Result<Self, Error> {
+        let file = Self::open(&file_name, &options)?;
+
+        let file_name: std::path::PathBuf = file_name.as_ref().to_path_buf();
+
+        Ok(Self { file, file_name, options })
+    }
+
+    pub fn reopen(&mut self) -> Result<&Self, Error> {
+        let file = Self::open(&self.file_name, &self.options)?;
+        self.file = file;
+        Ok(self)
+    }
+
+    fn open<P: AsRef<std::path::Path>>(
+        file_name: P,
+        options: &FileLogOptions,
+    ) -> Result<std::fs::File, Error> {
         let file = std::fs::OpenOptions::new()
             .read(options.read)
             .write(true)
@@ -76,7 +94,7 @@ impl FileLogger {
             nix::unistd::chown(file_name.as_ref(), Some(backup_user.uid), Some(backup_user.gid))?;
         }
 
-        Ok(Self { file, options })
+        Ok(file)
     }
 
     pub fn log<S: AsRef<str>>(&mut self, msg: S) {
@@ -88,15 +106,21 @@ impl FileLogger {
             stdout.write_all(b"\n").unwrap();
         }
 
-        let now = proxmox::tools::time::epoch_i64();
-        let rfc3339 = proxmox::tools::time::epoch_to_rfc3339(now).unwrap();
-
         let line = if self.options.prefix_time {
+            let now = proxmox::tools::time::epoch_i64();
+            let rfc3339 = match proxmox::tools::time::epoch_to_rfc3339(now) {
+                Ok(rfc3339) => rfc3339,
+                Err(_) => "1970-01-01T00:00:00Z".into(), // for safety, should really not happen!
+            };
             format!("{}: {}\n", rfc3339, msg)
         } else {
             format!("{}\n", msg)
         };
-        self.file.write_all(line.as_bytes()).unwrap();
+        if let Err(err) = self.file.write_all(line.as_bytes()) {
+            // avoid panicking, log methods should not do that
+            // FIXME: or, return result???
+            eprintln!("error writing to log file - {}", err);
+        }
     }
 }
 

src/tools/logrotate.rs

@@ -92,24 +92,24 @@ impl LogRotate {
         if filenames.is_empty() {
             return Ok(()); // no file means nothing to rotate
         }
+        let count = filenames.len() + 1;
 
         let mut next_filename = self.base_path.clone().canonicalize()?.into_os_string();
         next_filename.push(format!(".{}", filenames.len()));
-
+        if self.compress && count > 2 {
-        filenames.push(PathBuf::from(next_filename));
+            next_filename.push(".zst");
-        let count = filenames.len();
-
-        for i in (0..count-1).rev() {
-            rename(&filenames[i], &filenames[i+1])?;
         }
 
-        if self.compress {
+        filenames.push(PathBuf::from(next_filename));
-            for i in 2..count {
+
-                if filenames[i].extension().unwrap_or(std::ffi::OsStr::new("")) != "zst" {
+        for i in (0..count-1).rev() {
-                    let mut target = filenames[i].clone().into_os_string();
+            if self.compress
-                    target.push(".zstd");
+                && filenames[i+0].extension().unwrap_or(std::ffi::OsStr::new("")) != "zst"
-                    Self::compress(&filenames[i], &target.into(), &options)?;
+                && filenames[i+1].extension().unwrap_or(std::ffi::OsStr::new("")) == "zst"
-                }
+            {
+                Self::compress(&filenames[i], &filenames[i+1], &options)?;
+            } else {
+                rename(&filenames[i], &filenames[i+1])?;
             }
         }
 

tests/worker-task-abort.rs

@@ -42,8 +42,10 @@ fn worker_task_abort() -> Result<(), Error> {
     let mut rt = tokio::runtime::Runtime::new().unwrap();
     rt.block_on(async move {
 
+        let mut commando_sock = server::CommandoSocket::new(server::our_ctrl_sock());
+
         let init_result: Result<(), Error> = try_block!({
-            server::create_task_control_socket()?;
+            server::register_task_control_commands(&mut commando_sock)?;
             server::server_state_init()?;
             Ok(())
         });

www/Dashboard.js

@@ -63,7 +63,9 @@ Ext.define('PBS.Dashboard', {
 	updateSubscription: function(store, records, success) {
 	    if (!success) { return; }
 	    let me = this;
-	    let subStatus = records[0].data.status === 'Active' ? 2 : 0; // 2 = all good, 1 = different leves, 0 = none
+	    let status = records[0].data.status || 'unknown';
+	    // 2 = all good, 1 = different leves, 0 = none
+	    let subStatus = status.toLowerCase() === 'active' ? 2 : 0;
 	    me.lookup('subscription').setSubStatus(subStatus);
 	},
 
@@ -230,8 +232,9 @@ Ext.define('PBS.Dashboard', {
 		model: 'proxmox-tasks',
 		proxy: {
 		    type: 'proxmox',
-		    url: '/api2/json/status/tasks',
+		    url: '/api2/json/nodes/localhost/tasks',
 		    extraParams: {
+			limit: 0,
 			since: '{sinceEpoch}',
 		    },
 		},

www/MainView.js

@@ -236,9 +236,19 @@ Ext.define('PBS.MainView', {
 		    iconCls: 'fa fa-user',
 		    menu: [
 			{
-			    reference: 'logoutButton',
+			    iconCls: 'fa fa-language',
+			    text: gettext('Language'),
+			    reference: 'languageButton',
+			    handler: () => Ext.create('Proxmox.window.LanguageEditWindow', {
+				cookieName: 'PBSLangCookie',
+				autoShow: true,
+			    }),
+			},
+			'-',
+			{
 			    iconCls: 'fa fa-sign-out',
 			    text: gettext('Logout'),
+			    reference: 'logoutButton',
 			},
 		    ],
 		},

www/Makefile

@@ -5,8 +5,10 @@ IMAGES := \
 	images/proxmox_logo.png
 
 JSSRC=							\
+	Utils.js					\
 	form/UserSelector.js				\
 	form/TokenSelector.js				\
+	form/AuthidSelector.js				\
 	form/RemoteSelector.js				\
 	form/DataStoreSelector.js			\
 	form/CalendarEvent.js				\
@@ -21,11 +23,13 @@ JSSRC=							\
 	config/VerifyView.js				\
 	window/ACLEdit.js				\
 	window/BackupFileDownloader.js			\
+	window/BackupGroupChangeOwner.js		\
 	window/CreateDirectory.js			\
 	window/DataStoreEdit.js				\
 	window/FileBrowser.js				\
 	window/NotesEdit.js				\
 	window/RemoteEdit.js				\
+	window/NotifyOptions.js				\
 	window/SyncJobEdit.js				\
 	window/UserEdit.js				\
 	window/UserPassword.js				\
@@ -36,20 +40,22 @@ JSSRC=							\
 	dashboard/LongestTasks.js			\
 	dashboard/RunningTasks.js			\
 	dashboard/TaskSummary.js			\
-	Utils.js					\
+	panel/Tasks.js					\
-	AccessControlPanel.js				\
+	panel/XtermJsConsole.js				\
+	panel/AccessControl.js				\
 	ZFSList.js					\
 	DirectoryList.js				\
 	LoginView.js					\
 	VersionInfo.js					\
 	SystemConfiguration.js				\
 	Subscription.js					\
-	DataStoreSummary.js				\
+	datastore/Summary.js				\
-	DataStoreNotes.js				\
+	datastore/Notes.js				\
-	DataStorePruneAndGC.js				\
+	datastore/PruneAndGC.js				\
-	DataStorePrune.js				\
+	datastore/Prune.js				\
-	DataStoreContent.js				\
+	datastore/Content.js				\
-	DataStorePanel.js				\
+	datastore/OptionView.js				\
+	datastore/Panel.js				\
 	ServerStatus.js					\
 	ServerAdministration.js				\
 	Dashboard.js					\