bump version to 0.9.7-1

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "proxmox-backup"
-version = "0.9.1"
+version = "0.9.7"
 authors = ["Dietmar Maurer <dietmar@proxmox.com>"]
 edition = "2018"
 license = "AGPL-3"
@@ -29,7 +29,7 @@ hyper = "0.13.6"
 lazy_static = "1.4"
 libc = "0.2"
 log = "0.4"
-nix = "0.16"
+nix = "0.19"
 num-traits = "0.2"
 once_cell = "1.3.1"
 openssl = "0.10"
@@ -38,7 +38,7 @@ pam-sys = "0.5"
 percent-encoding = "2.1"
 pin-utils = "0.1.0"
 pathpatterns = "0.1.2"
-proxmox = { version = "0.4.3", features = [ "sortable-macro", "api-macro", "websocket" ] }
+proxmox = { version = "0.7.0", features = [ "sortable-macro", "api-macro", "websocket" ] }
 #proxmox = { git = "git://git.proxmox.com/git/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
 #proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro", "websocket" ] }
 proxmox-fuse = "0.1.0"

Makefile

@@ -19,7 +19,8 @@ USR_SBIN := \
 SERVICE_BIN := \
 	proxmox-backup-api \
 	proxmox-backup-banner \
-	proxmox-backup-proxy
+	proxmox-backup-proxy \
+	proxmox-daily-update \
 
 ifeq ($(BUILD_MODE), release)
 CARGO_BUILD_ARGS += --release

debian/changelog

@@ -1,3 +1,215 @@
+rust-proxmox-backup (0.9.7-1) unstable; urgency=medium
+
+  * ui: add remote store selector
+
+  * tools/daemon: fix reload with open connections
+
+  * pxar/create: fix endless loop for shrinking files
+
+  * pxar/create: handle ErrorKind::Interrupted for file reads
+
+  * ui: add action-button for changing backup group owner
+
+  * docs: add interactive prune simulator
+
+  * verify: fix unprivileged verification jobs
+
+  * tasks: allow access to job tasks
+
+  * drop internal 'backup@pam' owner, sync jobs need to set a explicit owner
+
+  * api: datastore: allow to set "verify-new" option over API
+
+  * ui: datastore: add Options tab, allowing one to change per-datastore
+    notification and verify-new options
+
+  * docs: scroll navigation bar to current active section
+
+ -- Proxmox Support Team <support@proxmox.com>  Mon, 09 Nov 2020 07:36:58 +0100
+
+rust-proxmox-backup (0.9.6-1) unstable; urgency=medium
+
+  * fix #3106: improve queueing new incoming connections
+
+  * fix #2870: sync: ensure a updated ticket is used, if available
+
+  * proxy: log if there are too many open connections
+
+  * ui: SyncJobEdit: fix sending 'delete' values on SyncJob creation
+
+  * datastore config: allow to configure who receives job notify emails
+
+  * ui: fix task description for log rotate
+
+  * proxy: also rotate auth.log file
+
+  * ui: add shell panel under administration
+
+  * ui: sync jobs: only set default schedule when creating new jobs and some
+    other small fixes
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 04 Nov 2020 19:12:57 +0100
+
+rust-proxmox-backup (0.9.5-1) unstable; urgency=medium
+
+  * ui: user menu: allow one to change the language while staying logged in
+
+  * proxmox-backup-manager: add subscription commands
+
+  * server/rest: also accept = as token separator
+
+  * privs: allow reading snapshot notes with Datastore.Audit
+
+  * privs: enforce Datastore.Modify|Backup to set backup notes
+
+  * verify: introduce and use new Datastore.Verify privilege
+
+  * docs: add API tokens to documentation
+
+  * ui: various smaller layout and icon improvements
+
+  * api: implement apt pkg cache for caching pending updates
+
+  * api: apt: implement support to send notification email on new updates
+
+  * add daily update and maintenance task
+
+  * fix #2864: add owner option to sync
+
+  * sync: allow sync for non-superusers under special conditions
+
+  * config: support depreacated netmask when parsing interfaces file
+
+  * server: implement access log rotation with re-open via command socket
+
+  * garbage collect: improve index error messages
+
+  * fix #3039: use the same ID regex for info and api
+
+  * ui: administration: allow extensive filtering of the worker task
+
+  * report: add api endpoint and function to generate report
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 03 Nov 2020 17:41:17 +0100
+
+rust-proxmox-backup (0.9.4-2) unstable; urgency=medium
+
+  * make postinst (update) script more resilient
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 29 Oct 2020 20:09:30 +0100
+
+rust-proxmox-backup (0.9.4-1) unstable; urgency=medium
+
+  * implement API-token
+
+  * client/remote: allow using API-token + secret
+
+  * ui/cli: implement API-token management interface and commands
+
+  * ui: add widget to view the effective permissions of a user or token
+
+  * ui: datastore summary: handle error when havin zero snapshot of any type
+
+  * ui: move user, token and permissions into an access control tab panel
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 29 Oct 2020 17:19:13 +0100
+
+rust-proxmox-backup (0.9.3-1) unstable; urgency=medium
+
+  * fix #2998: encode mtime as i64 instead of u64
+
+  * GC: log the number of leftover bad chunks we could not yet cleanup, as no
+    valid one replaced them. Also log deduplication factor.
+
+  * send sync job status emails
+
+  * api: datstore status: introduce proper structs and restore compatibility
+    to 0.9.1
+
+  * ui: drop id field from verify/sync add window, they are now seen as internal
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 29 Oct 2020 14:58:13 +0100
+
+rust-proxmox-backup (0.9.2-2) unstable; urgency=medium
+
+  * rework server web-interface, move more datastore related panels as tabs
+    inside the datastore view
+
+  * prune: never fail, just warn about failed removals
+
+  * prune/forget: skip snapshots with open readers (restore, verification)
+
+  * datastore: always ensure to remove individual snapshots before their group
+
+  * pxar: fix relative '!' rules in .pxarexclude
+
+  * pxar: anchor pxarexcludes starting with a slash
+
+  * GC: mark phase: ignore vanished index files
+
+  * server/rest: forward real client IP on proxied request and log it in
+    failed authentication requests
+
+  * server: rest: implement max URI path and query length request limits
+
+  * server/rest: implement request access log and log the query part of
+    URL and the user agent
+
+  * api: access: log to separate file, use syslog to errors only to reduce
+    syslog spam
+
+  * client: set HTTP connect timeout to 10 seconds
+
+  * client: sent TCP keep-alive after 2 minutes instead of the Linux default
+    of two hours.
+
+  * CLI completion: fix ACL path completion
+
+  * fix #2988: allow one to enable automatic verification after finishing a
+    snapshot, can be controlled as a per-datastore option
+
+  * various log-rotation improvements
+
+  * proxmox-backup-client: use HumanByte to render snapshot size
+
+  * paperkey: use svg as image format to provide better scalability
+
+  * backup: avoid Transport endpoint is not connected error
+
+  * fix #3038: check user before renewing ticket
+
+  * ui/tools: add zip module and allow to download an archive directory as a zip
+
+  * ui and api: add verification job config, allowing to schedule more
+    flexible jobs, filtering out already and/or recently verified snapshots
+    NOTE: the previous simple "verify all" schedule was dropped from the
+    datastore content, and does *not* gets migrated to the new job config.
+
+  * tasks: use systemd escape to decode/encode the task worker ID, avoiding
+    some display problems with problematic characters
+
+  * fix #2934: list also new to-be-installed packages in updates
+
+  * apt: add /changelog API call similar to PVE
+
+  * api: add world accessible ping dummy endpoint, to cheaply check for a
+    running PBS instance.
+
+  * ui: add datastore summary panel and move Statistics into it
+
+  * ui: navigation: add 'Add Datastore' button below datastore list
+
+  * ui: datastore panel: save and restore selected tab statefully
+
+  * send notification mails to email of root@pam account for GC and verify
+    jobs
+
+  * ui: datastore: use simple V. for verify action button
+
+  * ui: datastore: show snapshot manifest comment and allow to edit them
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 28 Oct 2020 23:05:41 +0100
+
 rust-proxmox-backup (0.9.1-1) unstable; urgency=medium
 
   * TLS speedups (use SslAcceptor::mozilla_intermediate_v5)
@@ -16,7 +228,7 @@ rust-proxmox-backup (0.9.1-1) unstable; urgency=medium
 
   * add "Build" section to README.rst
 
-  * reader: actually allow users to downlod their own backups
+  * reader: actually allow users to download their own backups
 
   * reader: track index chunks and limit access
 
@@ -38,7 +250,7 @@ rust-proxmox-backup (0.9.1-1) unstable; urgency=medium
 
   * ui: Dashboard/TaskSummary: add Verifies to the Summary
 
-  * ui: implment task history limit and make it configurable
+  * ui: implement task history limit and make it configurable
 
   * docs: installation: add system requirements section
 

debian/control

@@ -24,7 +24,7 @@ Build-Depends: debhelper (>= 11),
  librust-lazy-static-1+default-dev (>= 1.4-~~),
  librust-libc-0.2+default-dev,
  librust-log-0.4+default-dev,
- librust-nix-0.16+default-dev,
+ librust-nix-0.19+default-dev,
  librust-nom-5+default-dev (>= 5.1-~~),
  librust-num-traits-0.2+default-dev,
  librust-once-cell-1+default-dev (>= 1.3.1-~~),
@@ -34,10 +34,10 @@ Build-Depends: debhelper (>= 11),
  librust-pathpatterns-0.1+default-dev (>= 0.1.2-~~),
  librust-percent-encoding-2+default-dev (>= 2.1-~~),
  librust-pin-utils-0.1+default-dev,
- librust-proxmox-0.4+api-macro-dev (>= 0.4.3-~~),
- librust-proxmox-0.4+default-dev (>= 0.4.3-~~),
- librust-proxmox-0.4+sortable-macro-dev (>= 0.4.3-~~),
- librust-proxmox-0.4+websocket-dev (>= 0.4.3-~~),
+ librust-proxmox-0.7+api-macro-dev,
+ librust-proxmox-0.7+default-dev,
+ librust-proxmox-0.7+sortable-macro-dev,
+ librust-proxmox-0.7+websocket-dev,
  librust-proxmox-fuse-0.1+default-dev,
  librust-pxar-0.6+default-dev (>= 0.6.1-~~),
  librust-pxar-0.6+futures-io-dev (>= 0.6.1-~~),
@@ -107,7 +107,7 @@ Depends: fonts-font-awesome,
          pbs-i18n,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
-         proxmox-widget-toolkit (>= 2.3-1),
+         proxmox-widget-toolkit (>= 2.3-6),
          pve-xtermjs (>= 4.7.0-1),
          smartmontools,
          ${misc:Depends},

debian/control.in

@@ -7,7 +7,7 @@ Depends: fonts-font-awesome,
          pbs-i18n,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
-         proxmox-widget-toolkit (>= 2.3-1),
+         proxmox-widget-toolkit (>= 2.3-6),
          pve-xtermjs (>= 4.7.0-1),
          smartmontools,
          ${misc:Depends},

debian/postinst

@@ -15,10 +15,24 @@ case "$1" in
 	fi
 	deb-systemd-invoke $_dh_action proxmox-backup.service proxmox-backup-proxy.service >/dev/null || true
 
+	# FIXME: Remove with 1.1
+	if test -n "$2"; then
+		if dpkg --compare-versions "$2" 'lt' '0.9.4-1'; then
+			if grep -s -q  -P -e '^\s+verify-schedule ' /etc/proxmox-backup/datastore.cfg; then
+				echo "NOTE: drop all verify schedules from datastore config."
+				echo "You can now add more flexible verify jobs"
+				flock -w 30 /etc/proxmox-backup/.datastore.lck \
+				    sed -i '/^\s\+verify-schedule /d' /etc/proxmox-backup/datastore.cfg || true
+			fi
+		fi
+		if dpkg --compare-versions "$2" 'le' '0.9.5-1'; then
+			chown --quiet backup:backup /var/log/proxmox-backup/api/auth.log || true
+		fi
+	fi
 	# FIXME: Remove in future version once we're sure no broken entries remain in anyone's files
 	if grep -q -e ':termproxy::[^@]\+: ' /var/log/proxmox-backup/tasks/active; then
 	    echo "Fixing up termproxy user id in task log..."
-	    flock -w 30 /var/log/proxmox-backup/tasks/active.lock sed -i 's/:termproxy::\([^@]\+\): /:termproxy::\1@pam: /' /var/log/proxmox-backup/tasks/active
+	    flock -w 30 /var/log/proxmox-backup/tasks/active.lock sed -i 's/:termproxy::\([^@]\+\): /:termproxy::\1@pam: /' /var/log/proxmox-backup/tasks/active || true
 	fi
     ;;
 

debian/prerm

@@ -6,5 +6,6 @@ set -e
 
 # modeled after dh_systemd_start output
 if [ -d /run/systemd/system ] && [ "$1" = remove ]; then
-    deb-systemd-invoke stop 'proxmox-backup-banner.service' 'proxmox-backup-proxy.service' 'proxmox-backup.service' >/dev/null || true
+    deb-systemd-invoke stop 'proxmox-backup-banner.service' 'proxmox-backup-proxy.service' \
+        'proxmox-backup.service' 'proxmox-backup-daily-update.timer' >/dev/null || true
 fi

debian/proxmox-backup-docs.links

@@ -1 +1,2 @@
 /usr/share/doc/proxmox-backup/proxmox-backup.pdf /usr/share/doc/proxmox-backup/html/proxmox-backup.pdf
+/usr/share/javascript/extjs /usr/share/doc/proxmox-backup/html/prune-simulator/extjs

debian/proxmox-backup-server.install

@@ -1,10 +1,13 @@
 etc/proxmox-backup-proxy.service /lib/systemd/system/
 etc/proxmox-backup.service /lib/systemd/system/
 etc/proxmox-backup-banner.service /lib/systemd/system/
+etc/proxmox-backup-daily-update.service /lib/systemd/system/
+etc/proxmox-backup-daily-update.timer /lib/systemd/system/
 etc/pbstest-beta.list /etc/apt/sources.list.d/
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-api
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-proxy
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-banner
+usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-daily-update
 usr/sbin/proxmox-backup-manager
 usr/share/javascript/proxmox-backup/index.hbs
 usr/share/javascript/proxmox-backup/css/ext6-pbs.css

debian/rules

@@ -38,6 +38,7 @@ override_dh_auto_install:
 	    LIBDIR=/usr/lib/$(DEB_HOST_MULTIARCH)
 
 override_dh_installsystemd:
+	dh_installsystemd -pproxmox-backup-server  proxmox-backup-daily-update.timer
 	# note: we start/try-reload-restart services manually in postinst
 	dh_installsystemd --no-start --no-restart-after-upgrade
 

docs/Makefile

@@ -14,6 +14,10 @@ MANUAL_PAGES := 			\
 	proxmox-backup-client.1		\
 	proxmox-backup-manager.1
 
+PRUNE_SIMULATOR_FILES := 					\
+	prune-simulator/index.html				\
+	prune-simulator/documentation.html			\
+	prune-simulator/prune-simulator.js
 
 # Sphinx documentation setup
 SPHINXOPTS    =
@@ -74,10 +78,11 @@ onlinehelpinfo:
 	@echo "Build finished. OnlineHelpInfo.js is in $(BUILDDIR)/scanrefs."
 
 .PHONY: html
-html: ${GENERATED_SYNOPSIS} images/proxmox-logo.svg custom.css conf.py
+html: ${GENERATED_SYNOPSIS} images/proxmox-logo.svg custom.css conf.py ${PRUNE_SIMULATOR_FILES}
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
-	cp images/proxmox-logo.svg $(BUILDDIR)/html/_static/
-	cp custom.css $(BUILDDIR)/html/_static/
+	install -m 0644 custom.js custom.css images/proxmox-logo.svg $(BUILDDIR)/html/_static/
+	install -dm 0755 $(BUILDDIR)/html/prune-simulator
+	install -m 0644 ${PRUNE_SIMULATOR_FILES} $(BUILDDIR)/html/prune-simulator
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 

docs/backup-client.rst

@@ -12,31 +12,31 @@ on the backup server.
 
   [[username@]server[:port]:]datastore
 
-The default value for ``username`` is ``root@pam``.  If no server is specified,
+The default value for ``username`` is ``root@pam``. If no server is specified,
 the default is the local host (``localhost``).
 
 You can specify a port if your backup server is only reachable on a different
 port (e.g. with NAT and port forwarding).
 
-Note that if the server is an IPv6 address, you have to write it with
-square brackets (e.g. [fe80::01]).
+Note that if the server is an IPv6 address, you have to write it with square
+brackets (for example, `[fe80::01]`).
 
-You can pass the repository with the ``--repository`` command
-line option, or by setting the ``PBS_REPOSITORY`` environment
-variable.
+You can pass the repository with the ``--repository`` command line option, or
+by setting the ``PBS_REPOSITORY`` environment variable.
 
 Here some examples of valid repositories and the real values
 
-================================ ============ ================== ===========
-Example                          User         Host:Port          Datastore
-================================ ============ ================== ===========
-mydatastore                      ``root@pam`` localhost:8007     mydatastore
-myhostname:mydatastore           ``root@pam`` myhostname:8007    mydatastore
-user@pbs@myhostname:mydatastore  ``user@pbs`` myhostname:8007    mydatastore
-192.168.55.55:1234:mydatastore   ``root@pam`` 192.168.55.55:1234 mydatastore
-[ff80::51]:mydatastore           ``root@pam`` [ff80::51]:8007    mydatastore
-[ff80::51]:1234:mydatastore      ``root@pam`` [ff80::51]:1234    mydatastore
-================================ ============ ================== ===========
+================================ ================== ================== ===========
+Example                          User               Host:Port          Datastore
+================================ ================== ================== ===========
+mydatastore                      ``root@pam``       localhost:8007     mydatastore
+myhostname:mydatastore           ``root@pam``       myhostname:8007    mydatastore
+user@pbs@myhostname:mydatastore  ``user@pbs``       myhostname:8007    mydatastore
+user\@pbs!token@host:store       ``user@pbs!token`` myhostname:8007    mydatastore
+192.168.55.55:1234:mydatastore   ``root@pam``       192.168.55.55:1234 mydatastore
+[ff80::51]:mydatastore           ``root@pam``       [ff80::51]:8007    mydatastore
+[ff80::51]:1234:mydatastore      ``root@pam``       [ff80::51]:1234    mydatastore
+================================ ================== ================== ===========
 
 Environment Variables
 ---------------------
@@ -45,16 +45,16 @@ Environment Variables
   The default backup repository.
 
 ``PBS_PASSWORD``
-  When set, this value is used for the password required for the
-  backup server.
+  When set, this value is used for the password required for the backup server.
+  You can also set this to a API token secret.
 
 ``PBS_ENCRYPTION_PASSWORD``
   When set, this value is used to access the secret encryption key (if
   protected by password).
 
 ``PBS_FINGERPRINT`` When set, this value is used to verify the server
-  certificate (only used if the system CA certificates cannot
-  validate the certificate).
+  certificate (only used if the system CA certificates cannot validate the
+  certificate).
 
 
 Output Format
@@ -246,6 +246,8 @@ Restoring this backup will result in:
     .  ..  file2
 
 
+.. _encryption:
+
 Encryption
 ----------
 
@@ -533,6 +535,29 @@ To remove the ticket, issue a logout:
   # proxmox-backup-client logout
 
 
+.. _changing-backup-owner:
+
+Changing the Owner of a Backup Group
+------------------------------------
+
+By default, the owner of a backup group is the user which was used to originally
+create that backup group (or in the case of sync jobs, ``root@pam``). This
+means that if a user ``mike@pbs`` created a backup, another user ``john@pbs``
+can not be used to create backups in that same backup group.  In case you want
+to change the owner of a backup, you can do so with the below command, using a
+user that has ``Datastore.Modify`` privileges on the datastore.
+
+.. code-block:: console
+
+  # proxmox-backup-client change-owner vm/103 john@pbs
+
+This can also be done from within the web interface, by navigating to the
+`Content` section of the datastore that contains the backup group and
+selecting the user icon under the `Actions` column. Common cases for this could
+be to change the owner of a sync job from ``root@pam``, or to repurpose a
+backup group.
+
+
 .. _backup-pruning:
 
 Pruning and Removing Backups

docs/conf.py

@@ -171,6 +171,7 @@ html_theme_options = {
     'extra_nav_links': {
         'Proxmox Homepage': 'https://proxmox.com',
         'PDF': 'proxmox-backup.pdf',
+        'Prune Simulator' : 'prune-simulator/index.html',
     },
 
     'sidebar_width': '320px',
@@ -228,6 +229,10 @@ html_favicon = 'images/favicon.ico'
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+html_js_files = [
+    'custom.js',
+]
+
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.

docs/custom.js

@@ -0,0 +1,7 @@
+window.addEventListener('DOMContentLoaded', (event) => {
+    let activeSection = document.querySelector("a.current");
+    if (activeSection) {
+        // https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollIntoView
+        activeSection.scrollIntoView({ block: 'center' });
+    }
+});

docs/gui.rst

@@ -4,7 +4,7 @@ Graphical User Interface
 Proxmox Backup Server offers an integrated, web-based interface to manage the
 server. This means that you can carry out all administration tasks through your
 web browser, and that you don't have to worry about installing extra management
-tools. The web interface also provides a built in console, so if you prefer the
+tools. The web interface also provides a built-in console, so if you prefer the
 command line or need some extra control, you have this option.
 
 The web interface can be accessed via https://youripaddress:8007. The default
@@ -28,7 +28,6 @@ Login
 -----
 
 .. image:: images/screenshots/pbs-gui-login-window.png
-  :width: 250
   :align: right
   :alt: PBS login window
 
@@ -44,14 +43,13 @@ GUI Overview
 ------------
 
 .. image:: images/screenshots/pbs-gui-dashboard.png
-  :width: 250
   :align: right
   :alt: PBS GUI Dashboard
 
 The Proxmox Backup Server web interface consists of 3 main sections:
 
 * **Header**: At the top. This shows version information, and contains buttons to view
-  documentation, monitor running tasks, and logout.
+  documentation, monitor running tasks, set the language and logout.
 * **Sidebar**: On the left. This contains the configuration options for
   the server.
 * **Configuration Panel**: In the center. This contains the control interface for the
@@ -79,18 +77,17 @@ Configuration
 The Configuration section contains some system configuration options, such as
 time and network configuration. It also contains the following subsections:
 
-* **User Management**: Add users and manage accounts
-* **Permissions**: Manage permissions for various users
+* **Access Control**: Add and manage users, API tokens, and the permissions
+  associated with these items
 * **Remotes**: Add, edit and remove remotes (see :term:`Remote`)
-* **Sync Jobs**: Manage and run sync jobs to remotes
-* **Subscription**: Upload a subscription key and view subscription status
+* **Subscription**: Upload a subscription key, view subscription status and
+  access a text-based system report.
 
 
 Administration
 ^^^^^^^^^^^^^^
 
 .. image:: images/screenshots/pbs-gui-administration-serverstatus.png
-  :width: 250
   :align: right
   :alt: Administration: Server Status overview
 
@@ -105,7 +102,6 @@ tasks and information. These are:
 * **Tasks**: Task history with multiple filter options
 
 .. image:: images/screenshots/pbs-gui-disks.png
-  :width: 250
   :align: right
   :alt: Administration: Disks
 
@@ -120,16 +116,21 @@ The administration menu item also contains a disk management subsection:
 Datastore
 ^^^^^^^^^
 
-.. image:: images/screenshots/pbs-gui-datastore.png
-  :width: 250
+.. image:: images/screenshots/pbs-gui-datastore-summary.png
   :align: right
   :alt: Datastore Configuration
 
-The Datastore section provides an interface for creating and managing
-datastores. It contains a subsection for each datastore on the system, in
-which you can use the top panel to view:
+The Datastore section contains interfaces for creating and managing
+datastores. It contains a button to create a new datastore on the server, as
+well as a subsection for each datastore on the system, in which you can use the
+top panel to view:
 
+* **Summary**: Access a range of datastore usage statistics
 * **Content**: Information on the datastore's backup groups and their respective
   contents
-* **Statistics**: Usage statistics for the datastore
-* **Permissions**: View and manage permissions for the datastore
+* **Prune & GC**: Schedule :ref:`pruning <backup-pruning>` and :ref:`garbage
+  collection <garbage-collection>` operations, and run garbage collection
+  manually
+* **Sync Jobs**: Create, manage and run :ref:`syncjobs` from remote servers
+* **Verify Jobs**: Create, manage and run :ref:`verification` jobs on the
+  datastore

docs/installation.rst

@@ -9,7 +9,7 @@ Debian_ from the provided package repository.
 
 .. include:: package-repositories.rst
 
-Server installation
+Server Installation
 -------------------
 
 The backup server stores the actual backed up data and provides a web based GUI
@@ -52,7 +52,7 @@ It includes the following:
    is used by default and all existing data is removed.
 
 
-Install `Proxmox Backup`_ server on Debian
+Install `Proxmox Backup`_ Server on Debian
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Proxmox ships as a set of Debian packages which can be installed on top of a
@@ -84,11 +84,11 @@ support, and a set of common and useful packages.
   when LVM_ or ZFS_ is used. The network configuration is completely up to you
   as well.
 
-.. note:: You can access the web interface of the Proxmox Backup Server with
-  your web browser, using HTTPS on port 8007. For example at
- ``https://<ip-or-dns-name>:8007``
+.. Note:: You can access the web interface of the Proxmox Backup Server with
+   your web browser, using HTTPS on port 8007. For example at
+   ``https://<ip-or-dns-name>:8007``
 
-Install Proxmox Backup server on `Proxmox VE`_
+Install Proxmox Backup Server on `Proxmox VE`_
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 After configuring the
@@ -104,14 +104,14 @@ After configuring the
    server to store backups. Should the hypervisor server fail, you can
    still access the backups.
 
-.. note::
-  You can access the web interface of the Proxmox Backup Server with your web
-  browser, using HTTPS on port 8007. For example at ``https://<ip-or-dns-name>:8007``
+.. Note:: You can access the web interface of the Proxmox Backup Server with
+   your web browser, using HTTPS on port 8007. For example at
+   ``https://<ip-or-dns-name>:8007``
 
-Client installation
+Client Installation
 -------------------
 
-Install `Proxmox Backup`_ client on Debian
+Install `Proxmox Backup`_ Client on Debian
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Proxmox ships as a set of Debian packages to be installed on

docs/introduction.rst

@@ -1,8 +1,8 @@
 Introduction
 ============
 
-What is Proxmox Backup Server
------------------------------
+What is Proxmox Backup Server?
+------------------------------
 
 Proxmox Backup Server is an enterprise-class, client-server backup software
 package that backs up :term:`virtual machine`\ s, :term:`container`\ s, and
@@ -10,12 +10,14 @@ physical hosts. It is specially optimized for the `Proxmox Virtual Environment`_
 platform and allows you to back up your data securely, even between remote
 sites, providing easy management with a web-based user interface.
 
-Proxmox Backup Server supports deduplication, compression, and authenticated
+It supports deduplication, compression, and authenticated
 encryption (AE_). Using :term:`Rust` as the implementation language guarantees high
 performance, low resource usage, and a safe, high-quality codebase.
 
-It features strong client-side encryption. Thus, it's possible to
-backup data to targets that are not fully trusted.
+Proxmox Backup uses state of the art cryptography for client communication and
+backup content :ref:`encryption <encryption>`. Encryption is done on the
+client side, making it safer to back up data to targets that are not fully
+trusted.
 
 
 Architecture
@@ -179,29 +181,28 @@ along with this program.  If not, see AGPL3_.
 History
 -------
 
-Backup is, and always was, as central aspect of IT administration.
-The need to recover from data loss is fundamental and increases with
+Backup is, and always has been, a central aspect of IT administration.
+The need to recover from data loss is fundamental and only increases with
 virtualization.
 
-Not surprisingly, we shipped a backup tool with Proxmox VE from the
-beginning. The tool is called ``vzdump`` and is able to make
+For this reason, we've been shipping a backup tool with Proxmox VE, from the
+beginning. This tool is called ``vzdump`` and is able to make
 consistent snapshots of running LXC containers and KVM virtual
 machines.
 
-But ``vzdump`` only allowed for full backups. While this is perfect
+However, ``vzdump`` only allows for full backups. While this is fine
 for small backups, it becomes a burden for users with large VMs. Both
-backup time and space usage was too large for this case, specially
-when Users want to keep many backups of the same VMs. We need
-deduplication and incremental backups to solve those problems.
+backup duration and storage usage are too high for this case, especially
+for users who want to keep many backups of the same VMs. To solve these
+problems, we needed to offer deduplication and incremental backups.
 
-Back in October 2018 development started. We had been looking into
+Back in October 2018, development started. We investigated
 several technologies and frameworks and finally decided to use
-:term:`Rust` as implementation language to provide high speed and
-memory efficiency. The 2018-edition of Rust seemed to be promising and
-useful for our requirements.
+:term:`Rust` as the implementation language, in order to provide high speed and
+memory efficiency. The 2018-edition of Rust seemed promising for our
+requirements.
 
-In July 2020 we released the first beta version of Proxmox Backup
-Server, followed by a first stable version in November 2020. With the
-support of incremental, fully deduplicated backups, Proxmox Backup
-significantly reduces the network load and saves valuable storage
-space.
+In July 2020, we released the first beta version of Proxmox Backup
+Server, followed by the first stable version in November 2020. With support for
+incremental, fully deduplicated backups, Proxmox Backup significantly reduces
+network load and saves valuable storage space.

docs/maintenance.rst

@@ -1,13 +1,148 @@
 Maintenance Tasks
 =================
 
+.. _maintenance_pruning:
+
+Pruning
+-------
+
+Prune lets you specify which backup snapshots you want to keep. The
+following retention options are available:
+
+``keep-last <N>``
+  Keep the last ``<N>`` backup snapshots.
+
+``keep-hourly <N>``
+  Keep backups for the last ``<N>`` hours. If there is more than one
+  backup for a single hour, only the latest is kept.
+
+``keep-daily <N>``
+  Keep backups for the last ``<N>`` days. If there is more than one
+  backup for a single day, only the latest is kept.
+
+``keep-weekly <N>``
+  Keep backups for the last ``<N>`` weeks. If there is more than one
+  backup for a single week, only the latest is kept.
+
+  .. note:: Weeks start on Monday and end on Sunday. The software
+     uses the `ISO week date`_ system and handles weeks at
+     the end of the year correctly.
+
+``keep-monthly <N>``
+  Keep backups for the last ``<N>`` months. If there is more than one
+  backup for a single month, only the latest is kept.
+
+``keep-yearly <N>``
+  Keep backups for the last ``<N>`` years. If there is more than one
+  backup for a single year, only the latest is kept.
+
+The retention options are processed in the order given above. Each option
+only covers backups within its time period. The next option does not take care
+of already covered backups. It will only consider older backups.
+
+Unfinished and incomplete backups will be removed by the prune command unless
+they are newer than the last successful backup. In this case, the last failed
+backup is retained.
+
+Prune Simulator
+^^^^^^^^^^^^^^^
+
+You can use the built-in `prune simulator <prune-simulator/index.html>`_
+to explore the effect of different retetion options with various backup
+schedules.
+
+Manual Pruning
+^^^^^^^^^^^^^^
+
+.. image:: images/screenshots/pbs-gui-datastore-content-prune-group.png
+  :target: _images/pbs-gui-datastore-content-prune-group.png
+  :align: right
+  :alt: Prune and garbage collection options
+
+To access pruning functionality for a specific backup group, you can use the
+prune command line option discussed in :ref:`backup-pruning`, or navigate to
+the **Content** tab of the datastore and click the scissors icon in the
+**Actions** column of the relevant backup group.
+
+Prune Schedules
+^^^^^^^^^^^^^^^
+
+To prune on a datastore level, scheduling options can be found under the
+**Prune & GC** tab of the datastore. Here you can set retention settings and
+edit the interval at which pruning takes place.
+
+.. image:: images/screenshots/pbs-gui-datastore-prunegc.png
+  :target: _images/pbs-gui-datastore-prunegc.png
+  :align: right
+  :alt: Prune and garbage collection options
+
+
+.. _maintenance_gc:
+
 Garbage Collection
 ------------------
 
 You can monitor and run :ref:`garbage collection <garbage-collection>` on the
 Proxmox Backup Server using the ``garbage-collection`` subcommand of
-``proxmox-backup-manager``. You can use the ``start`` subcommand to manually start garbage
-collection on an entire datastore and the ``status`` subcommand to see
-attributes relating to the :ref:`garbage collection <garbage-collection>`.
+``proxmox-backup-manager``. You can use the ``start`` subcommand to manually
+start garbage collection on an entire datastore and the ``status`` subcommand to
+see attributes relating to the :ref:`garbage collection <garbage-collection>`.
 
-.. todo:: Add section on verification
+This functionality can also be accessed in the GUI, by navigating to **Prune &
+GC** from the top panel. From here, you can edit the schedule at which garbage
+collection runs and manually start the operation.
+
+
+.. _verification:
+
+Verification
+------------
+
+.. image:: images/screenshots/pbs-gui-datastore-verifyjob-add.png
+  :target: _images/pbs-gui-datastore-verifyjob-add.png
+  :align: right
+  :alt: Adding a verify job
+
+Proxmox Backup offers various verification options to ensure that backup data is
+intact.  Verification is generally carried out through the creation of verify
+jobs. These are scheduled tasks that run verification at a given interval (see
+:ref:`calendar-events`). With these, you can set whether already verified
+snapshots are ignored, as well as set a time period, after which verified jobs
+are checked again. The interface for creating verify jobs can be found under the
+**Verify Jobs** tab of the datastore.
+
+.. Note:: It is recommended that you reverify all backups at least monthly, even
+  if a previous verification was successful. This is becuase physical drives
+  are susceptible to damage over time, which can cause an old, working backup
+  to become corrupted in a process known as `bit rot/data degradation
+  <https://en.wikipedia.org/wiki/Data_degradation>`_. It is good practice to
+  have a regularly recurring (hourly/daily) verification job, which checks new
+  and expired backups, then another weekly/monthly job that will reverify
+  everything. This way, there will be no surprises when it comes to restoring
+  data.
+
+Aside from using verify jobs, you can also run verification manually on entire
+datastores, backup groups, or snapshots. To do this, navigate to the **Content**
+tab of the datastore and either click *Verify All*, or select the *V.* icon from
+the *Actions* column in the table.
+
+.. _maintenance_notification:
+
+Notifications
+-------------
+
+Proxmox Backup Server can send you notification emails about automatically
+scheduled verification, garbage-collection and synchronization tasks results.
+
+By default, notifications are send to the email address configured for the
+`root@pam` user. You can set that user for each datastore.
+
+You can also change the level of notification received per task type, the
+following options are available:
+
+* Always: send a notification for any scheduled task, independent of the
+  outcome
+
+* Errors: send a notification for any scheduled task resulting in an error
+
+* Never: do not send any notification at all

docs/managing-remotes.rst

@@ -59,13 +59,13 @@ Sync Jobs
   :alt: Add a Sync Job
 
 Sync jobs are configured to pull the contents of a datastore on a **Remote** to
-a local datastore. You can manage sync jobs under **Configuration -> Sync Jobs**
-in the web interface, or using the ``proxmox-backup-manager sync-job`` command.
-The configuration information for sync jobs is stored at
-``/etc/proxmox-backup/sync.cfg``. To create a new sync job, click the add button
-in the GUI, or use the ``create`` subcommand. After creating a sync job, you can
-either start it manually on the GUI or provide it with a schedule (see
-:ref:`calendar-events`) to run regularly.
+a local datastore. You can manage sync jobs in the web interface, from the
+**Sync Jobs** tab of the datastore which you'd like to set one up for, or using
+the ``proxmox-backup-manager sync-job`` command.  The configuration information
+for sync jobs is stored at ``/etc/proxmox-backup/sync.cfg``. To create a new
+sync job, click the add button in the GUI, or use the ``create`` subcommand.
+After creating a sync job, you can either start it manually from the GUI or
+provide it with a schedule (see :ref:`calendar-events`) to run regularly.
 
 .. code-block:: console
 
@@ -79,4 +79,17 @@ either start it manually on the GUI or provide it with a schedule (see
   └────────────┴───────┴────────┴──────────────┴───────────┴─────────┘
   # proxmox-backup-manager sync-job remove pbs2-local
 
+For setting up sync jobs, the configuring user needs the following permissions:
 
+#. ``Remote.Read`` on the ``/remote/{remote}/{remote-store}`` path
+#. at least ``Datastore.Backup`` on the local target datastore (``/datastore/{store}``)
+
+If the ``remove-vanished`` option is set, ``Datastore.Prune`` is required on
+the local datastore as well. If the ``owner`` option is not set (defaulting to
+``root@pam``) or set to something other than the configuring user,
+``Datastore.Modify`` is required as well.
+
+.. note:: A sync job can only sync backup groups that the configured remote's
+  user/API token can read. If a remote is configured with a user/API token that
+  only has ``Datastore.Backup`` privileges, only the limited set of accessible
+  snapshots owned by that user/API token can be synced.

docs/prune-simulator/documentation.html

@@ -0,0 +1,102 @@
+<!DOCTYPE html>
+<html>
+<head>
+<style>
+    /* similar to sphinx alabaster theme ones */
+    body {
+        max-width: 90ch;
+        margin-left: 2ch;
+        margin-right: 2ch;
+        line-height: 1.4em;
+        /* avoid the very high contrast of black on white, tone it down a bit */
+        color: #3E4349;
+        hyphens: auto;
+        text-align: left;
+        font-family: 'Open Sans', sans-serif;
+        font-size: 17px;
+    }
+    h1, h2, h3 {
+        font-family: Lato, sans-serif;
+        font-size: 150%;
+        line-height:1.2
+    }
+    tt, code {
+        background-color: #ecf0f3;
+        color: #222;
+    }
+    pre, tt, code {
+        font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace;
+        font-size: 0.9em;
+    }
+    div.note {
+        background-color: #EEE;
+        border: 1px solid #CCC;
+        margin: 10px 0;
+        padding: 0px 20px;
+    }
+    p.note-title {
+        font-weight: bolder;
+        padding: 0;
+        margin: 10px 0 0 0;
+    }
+    div.note > p.last {
+        margin: 5px 0 10px 0;
+    }
+</style>
+</head>
+<body>
+<p>A simulator to experiment with different backup schedules and prune
+options.</p>
+
+<h3>Schedule</h3>
+<p>Select weekdays with the combobox and input hour and minute
+specification separated by a colon, i.e. <code>HOUR:MINUTE</code>. Each of
+<code>HOUR</code> and <code>MINUTE</code> can be either a single value or
+one of the following:</p>
+<ul class="simple">
+<li>a comma-separated list: e.g., <code>01,02,03</code></li>
+<li>a range: e.g., <code>01..10</code></li>
+<li>a repetition: e.g, <code>05/10</code> (means starting at <code>5</code> every <code>10</code>)</li>
+<li>a combination of the above: e.g., <code>01,05..10,12/02</code></li>
+<li>a <code>*</code> for every possible value</li>
+</ul>
+
+<h3>Pruning</h3>
+<p>Prune lets you systematically delete older backups, retaining backups for
+the last given number of time intervals. The following retention options are
+available:</p>
+<dl class="docutils">
+<dt><code class="docutils literal notranslate"><span class="pre">keep-last</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> backup snapshots.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-hourly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> hours. If there is more than one
+backup for a single hour, only the latest is kept.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-daily</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> days. If there is more than one
+backup for a single day, only the latest is kept.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-weekly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> weeks. If there is more than one
+backup for a single week, only the latest is kept.
+<div class="last admonition note">
+<p class="note-title">Note:</p>
+<p class="last">Weeks start on Monday and end on Sunday. The software
+uses the <a class="reference external" href="https://en.wikipedia.org/wiki/ISO_week_date">ISO week date</a> system and handles weeks at
+the end of the year correctly.</p>
+</div>
+</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-monthly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> months. If there is more than one
+backup for a single month, only the latest is kept.</dd>
+<dt><code class="docutils literal notranslate"><span class="pre">keep-yearly</span> <span class="pre">&lt;N&gt;</span></code></dt>
+<dd>Keep backups for the last <code class="docutils literal notranslate"><span class="pre">&lt;N&gt;</span></code> years. If there is more than one
+backup for a single year, only the latest is kept.</dd>
+</dl>
+<p>The retention options are processed in the order given above. Each option
+only covers backups within its time period. The next option does not take care
+of already covered backups. It will only consider older backups.</p>
+<p>For example, in a week covered by <code>keep-weekly</code>, one backup is
+kept while all others are removed; <code>keep-monthly</code> then does not
+consider backups from that week anymore, even if part of the week is part of
+an earlier month.</p>
+</body>
+</html>

docs/prune-simulator/index.html

@@ -0,0 +1,41 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
+    <title>PBS Prune Simulator</title>
+
+    <link rel="stylesheet" type="text/css" href="extjs/theme-crisp/resources/theme-crisp-all.css">
+    <style>
+        .cal {
+            margin: 5px;
+        }
+        .cal-day {
+            vertical-align: top;
+            width: 150px;
+            border: #939393 1px solid;
+            color: #454545;
+        }
+        .cal-day-date {
+            border-bottom: #444 1px solid;
+            color: #000;
+        }
+        .strikethrough {
+            text-decoration: line-through;
+        }
+        .black {
+            color: #000;
+        }
+        .sun {
+            background-color: #ededed;
+        }
+        .first-of-month {
+            border-right: dashed black 4px;
+        }
+    </style>
+
+    <script type="text/javascript" src="extjs/ext-all.js"></script>
+    <script type="text/javascript" src="prune-simulator.js"></script>
+</head>
+<body></body>
+</html>

docs/prune-simulator/prune-simulator.js

@@ -0,0 +1,773 @@
+// FIXME: HACK! Makes scrolling in number spinner work again. fixed in ExtJS >= 6.1
+if (Ext.isFirefox) {
+    Ext.$eventNameMap.DOMMouseScroll = 'DOMMouseScroll';
+}
+
+Ext.onReady(function() {
+    const NOW = new Date();
+    const COLORS = {
+	'keep-last': 'orange',
+	'keep-hourly': 'purple',
+	'keep-daily': 'yellow',
+	'keep-weekly': 'green',
+	'keep-monthly': 'blue',
+	'keep-yearly': 'red',
+	'all zero': 'white',
+    };
+    const TEXT_COLORS = {
+	'keep-last': 'black',
+	'keep-hourly': 'white',
+	'keep-daily': 'black',
+	'keep-weekly': 'white',
+	'keep-monthly': 'white',
+	'keep-yearly': 'white',
+	'all zero': 'black',
+    };
+
+    Ext.define('PBS.prunesimulator.Documentation', {
+	extend: 'Ext.Panel',
+	alias: 'widget.prunesimulatorDocumentation',
+
+	html: '<iframe style="width:100%;height:100%;border:0px;" src="./documentation.html"/>',
+    });
+
+    Ext.define('PBS.prunesimulator.CalendarEvent', {
+	extend: 'Ext.form.field.ComboBox',
+	alias: 'widget.prunesimulatorCalendarEvent',
+
+	editable: true,
+
+	displayField: 'text',
+	valueField: 'value',
+	queryMode: 'local',
+
+	store: {
+	    field: ['value', 'text'],
+	    data: [
+		{ value: '0/2:00', text: "Every two hours" },
+		{ value: '0/6:00', text: "Every six hours" },
+		{ value: '2,22:30', text: "At 02:30 and 22:30" },
+		{ value: '00:00', text: "At 00:00" },
+		{ value: '08..17:00/30', text: "From 08:00 to 17:30 every 30 minutes" },
+		{ value: 'HOUR:MINUTE', text: "Custom schedule" },
+	    ],
+	},
+
+	tpl: [
+	    '<ul class="x-list-plain"><tpl for=".">',
+	    '<li role="option" class="x-boundlist-item">{text}</li>',
+	    '</tpl></ul>',
+	],
+
+	displayTpl: [
+	    '<tpl for=".">',
+	    '{value}',
+	    '</tpl>',
+	],
+    });
+
+    Ext.define('PBS.prunesimulator.DayOfWeekSelector', {
+	extend: 'Ext.form.field.ComboBox',
+	alias: 'widget.prunesimulatorDayOfWeekSelector',
+
+	editable: false,
+
+	displayField: 'text',
+	valueField: 'value',
+	queryMode: 'local',
+
+	store: {
+	    field: ['value', 'text'],
+	    data: [
+		{ value: 'mon', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[1]) },
+		{ value: 'tue', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[2]) },
+		{ value: 'wed', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[3]) },
+		{ value: 'thu', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[4]) },
+		{ value: 'fri', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[5]) },
+		{ value: 'sat', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[6]) },
+		{ value: 'sun', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[0]) },
+	    ],
+	},
+    });
+
+    Ext.define('pbs-prune-list', {
+	extend: 'Ext.data.Model',
+	fields: [
+	    {
+		name: 'backuptime',
+		type: 'date',
+		dateFormat: 'timestamp',
+	    },
+	    {
+		name: 'mark',
+		type: 'string',
+	    },
+	    {
+		name: 'keepName',
+		type: 'string',
+	    },
+	],
+    });
+
+    Ext.define('PBS.prunesimulator.PruneList', {
+	extend: 'Ext.panel.Panel',
+	alias: 'widget.prunesimulatorPruneList',
+
+	initComponent: function() {
+	    let me = this;
+
+	    if (!me.store) {
+		throw "no store specified";
+	    }
+
+	    me.items = [
+		{
+		    xtype: 'grid',
+		    store: me.store,
+		    border: false,
+		    columns: [
+			{
+			    header: 'Backup Time',
+			    dataIndex: 'backuptime',
+			    renderer: function(value, metaData, record) {
+				let text = Ext.Date.format(value, 'Y-m-d H:i:s');
+				if (record.data.mark === 'keep') {
+				    if (me.useColors) {
+					let bgColor = COLORS[record.data.keepName];
+					let textColor = TEXT_COLORS[record.data.keepName];
+					return '<div style="background-color: ' + bgColor + '; ' +
+							    'color: ' + textColor + ';">' + text + '</div>';
+				    } else {
+					return text;
+				    }
+				} else {
+				    return '<div style="text-decoration: line-through;">' + text + '</div>';
+				}
+			    },
+			    flex: 1,
+			    sortable: false,
+			},
+			{
+			    header: 'Keep (reason)',
+			    dataIndex: 'mark',
+			    renderer: function(value, metaData, record) {
+				if (record.data.mark === 'keep') {
+				    return 'keep (' + record.data.keepName + ')';
+				} else {
+				    return value;
+				}
+			    },
+			    width: 200,
+			    sortable: false,
+			},
+		    ],
+		},
+	    ];
+
+	    me.callParent();
+	},
+    });
+
+    Ext.define('PBS.prunesimulator.WeekTable', {
+	extend: 'Ext.panel.Panel',
+	alias: 'widget.prunesimulatorWeekTable',
+
+	reload: function() {
+	    let me = this;
+	    let backups = me.store.data.items;
+
+	    let html = '<table class="cal">';
+
+	    let now = new Date(NOW.getTime());
+	    let skip = 7 - parseInt(Ext.Date.format(now, 'N'), 10);
+	    let tableStartDate = Ext.Date.add(now, Ext.Date.DAY, skip);
+
+	    let bIndex = 0;
+
+	    for (let i = 0; bIndex < backups.length; i++) {
+		html += '<tr>';
+
+		for (let j = 0; j < 7; j++) {
+		    let date = Ext.Date.subtract(tableStartDate, Ext.Date.DAY, j + 7 * i);
+		    let currentDay = Ext.Date.format(date, 'd/m/Y');
+
+		    let dayOfWeekCls = Ext.Date.format(date, 'D').toLowerCase();
+		    let firstOfMonthCls = Ext.Date.format(date, 'd') === '01'
+		        ? 'first-of-month'
+		        : '';
+		    html += `<td class="cal-day ${dayOfWeekCls} ${firstOfMonthCls}">`;
+
+		    const isBackupOnDay = function(backup, day) {
+			return backup && Ext.Date.format(backup.data.backuptime, 'd/m/Y') === day;
+		    };
+
+		    let backup = backups[bIndex];
+
+		    html += '<table><tr>';
+		    html += `<th class="cal-day-date">${Ext.Date.format(date, 'D, d M Y')}</th>`;
+
+		    while (isBackupOnDay(backup, currentDay)) {
+			html += '<tr><td>';
+
+			let text = Ext.Date.format(backup.data.backuptime, 'H:i');
+			if (backup.data.mark === 'remove') {
+			    html += `<span class="strikethrough">${text}</span>`;
+			} else {
+			    text += ` (${backup.data.keepName})`;
+			    if (me.useColors) {
+				let bgColor = COLORS[backup.data.keepName];
+				let textColor = TEXT_COLORS[backup.data.keepName];
+				html += `<span style="background-color: ${bgColor};
+				    color: ${textColor};">${text}</span>`;
+			    } else {
+				html += `<span class="black">${text}</span>`;
+			    }
+			}
+			html += '</td></tr>';
+			backup = backups[++bIndex];
+		    }
+		    html += '</table>';
+		    html += '</div>';
+		    html += '</td>';
+		}
+
+		html += '</tr>';
+	    }
+
+	    me.setHtml(html);
+	},
+
+	initComponent: function() {
+	    let me = this;
+
+	    if (!me.store) {
+		throw "no store specified";
+	    }
+
+	    let reload = function() {
+		me.reload();
+	    };
+
+	    me.store.on("datachanged", reload);
+
+	    me.callParent();
+
+	    me.reload();
+	},
+    });
+
+    Ext.define('PBS.PruneSimulatorPanel', {
+	extend: 'Ext.panel.Panel',
+	alias: 'widget.prunesimulatorPanel',
+
+	viewModel: {
+	},
+
+	getValues: function() {
+	    let me = this;
+
+	    let values = {};
+
+	    Ext.Array.each(me.query('[isFormField]'), function(field) {
+		let data = field.getSubmitData();
+		Ext.Object.each(data, function(name, val) {
+		    values[name] = val;
+		});
+	    });
+
+	    return values;
+	},
+
+	controller: {
+	    xclass: 'Ext.app.ViewController',
+
+	    init: function(view) {
+		this.reloadFull(); // initial load
+		this.switchColor(true);
+	    },
+
+	    control: {
+		'field[fieldGroup=keep]': { change: 'reloadPrune' },
+	    },
+
+	    reloadFull: function() {
+		let me = this;
+		let view = me.getView();
+
+		let params = view.getValues();
+
+		let [hourSpec, minuteSpec] = params['schedule-time'].split(':');
+
+		if (!hourSpec || !minuteSpec) {
+		    Ext.Msg.alert('Error', 'Invalid schedule');
+		    return;
+		}
+
+		let matchTimeSpec = function(timeSpec, rangeMin, rangeMax) {
+		    let specValues = timeSpec.split(',');
+		    let matches = {};
+
+		    let assertValid = function(value) {
+			let num = Number(value);
+			if (isNaN(num)) {
+			    throw value + " is not an integer";
+			} else if (value < rangeMin || value > rangeMax) {
+			    throw "number '" + value + "' is not in the range '" + rangeMin + ".." + rangeMax + "'";
+			}
+			return num;
+		    };
+
+		    specValues.forEach(function(value) {
+			if (value.includes('..')) {
+			    let [start, end] = value.split('..');
+			    start = assertValid(start);
+			    end = assertValid(end);
+			    if (start > end) {
+				throw "interval start is bigger then interval end '" + start + " > " + end + "'";
+			    }
+			    for (let i = start; i <= end; i++) {
+				matches[i] = 1;
+			    }
+			} else if (value.includes('/')) {
+			    let [start, step] = value.split('/');
+			    start = assertValid(start);
+			    step = assertValid(step);
+			    for (let i = start; i <= rangeMax; i += step) {
+				matches[i] = 1;
+			    }
+			} else if (value === '*') {
+			    for (let i = rangeMin; i <= rangeMax; i++) {
+				matches[i] = 1;
+			    }
+			} else {
+			    value = assertValid(value);
+			    matches[value] = 1;
+			}
+		    });
+
+		    return Object.keys(matches);
+		};
+
+		let hours, minutes;
+
+		try {
+		    hours = matchTimeSpec(hourSpec, 0, 23);
+		    minutes = matchTimeSpec(minuteSpec, 0, 59);
+		} catch (err) {
+		    Ext.Msg.alert('Error', err);
+		    return;
+		}
+
+		let backups = me.populateFromSchedule(
+		    params['schedule-weekdays'],
+		    hours,
+		    minutes,
+		    params.numberOfWeeks,
+		);
+
+		me.pruneSelect(backups, params);
+
+		view.pruneStore.setData(backups);
+	    },
+
+	    reloadPrune: function() {
+		let me = this;
+		let view = me.getView();
+
+		let params = view.getValues();
+
+		let backups = [];
+		view.pruneStore.getData().items.forEach(function(item) {
+		    backups.push({
+			backuptime: item.data.backuptime,
+		    });
+		});
+
+		me.pruneSelect(backups, params);
+
+		view.pruneStore.setData(backups);
+	    },
+
+	    // backups are sorted descending by date
+	    populateFromSchedule: function(weekdays, hours, minutes, weekCount) {
+		let weekdayFlags = [
+		    weekdays.includes('sun'),
+		    weekdays.includes('mon'),
+		    weekdays.includes('tue'),
+		    weekdays.includes('wed'),
+		    weekdays.includes('thu'),
+		    weekdays.includes('fri'),
+		    weekdays.includes('sat'),
+		];
+
+		let todaysDate = new Date(NOW.getTime());
+
+		let timesOnSingleDay = [];
+
+		hours.forEach(function(hour) {
+		    minutes.forEach(function(minute) {
+			todaysDate.setHours(hour);
+			todaysDate.setMinutes(minute);
+			timesOnSingleDay.push(todaysDate.getTime());
+		    });
+		});
+
+		// ordering here and iterating backwards through days
+		// ensures that everything is ordered
+		timesOnSingleDay.sort(function(a, b) {
+		    return a < b;
+		});
+
+		let backups = [];
+
+		for (let i = 0; i < 7 * weekCount; i++) {
+		    let daysDate = Ext.Date.subtract(todaysDate, Ext.Date.DAY, i);
+		    let weekday = parseInt(Ext.Date.format(daysDate, 'w'), 10);
+		    if (weekdayFlags[weekday]) {
+			timesOnSingleDay.forEach(function(time) {
+			    backups.push({
+				backuptime: Ext.Date.subtract(new Date(time), Ext.Date.DAY, i),
+			    });
+			});
+		    }
+		}
+
+		return backups;
+	    },
+
+	    pruneMark: function(backups, keepCount, keepName, idFunc) {
+		if (!keepCount) {
+		    return;
+		}
+
+		let alreadyIncluded = {};
+		let newlyIncluded = {};
+		let newlyIncludedCount = 0;
+
+		let finished = false;
+
+		backups.forEach(function(backup) {
+		    let mark = backup.mark;
+		    let id = idFunc(backup);
+
+		    if (finished || alreadyIncluded[id]) {
+			return;
+		    }
+
+		    if (mark) {
+			if (mark === 'keep') {
+			    alreadyIncluded[id] = true;
+			}
+			return;
+		    }
+
+		    if (!newlyIncluded[id]) {
+			if (newlyIncludedCount >= keepCount) {
+			    finished = true;
+			    return;
+			}
+			newlyIncluded[id] = true;
+			newlyIncludedCount++;
+			backup.mark = 'keep';
+			backup.keepName = keepName;
+		    } else {
+			backup.mark = 'remove';
+		    }
+		});
+	    },
+
+	    // backups need to be sorted descending by date
+	    pruneSelect: function(backups, keepParams) {
+		let me = this;
+
+		if (Number(keepParams['keep-last']) +
+		    Number(keepParams['keep-hourly']) +
+		    Number(keepParams['keep-daily']) +
+		    Number(keepParams['keep-weekly']) +
+		    Number(keepParams['keep-monthly']) +
+		    Number(keepParams['keep-yearly']) === 0) {
+		    backups.forEach(function(backup) {
+			backup.mark = 'keep';
+			backup.keepName = 'all zero';
+		    });
+
+		    return;
+		}
+
+		me.pruneMark(backups, keepParams['keep-last'], 'keep-last', function(backup) {
+		    return backup.backuptime;
+		});
+		me.pruneMark(backups, keepParams['keep-hourly'], 'keep-hourly', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'H/d/m/Y');
+		});
+		me.pruneMark(backups, keepParams['keep-daily'], 'keep-daily', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'd/m/Y');
+		});
+		me.pruneMark(backups, keepParams['keep-weekly'], 'keep-weekly', function(backup) {
+		    // ISO-8601 week and week-based year
+		    return Ext.Date.format(backup.backuptime, 'W/o');
+		});
+		me.pruneMark(backups, keepParams['keep-monthly'], 'keep-monthly', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'm/Y');
+		});
+		me.pruneMark(backups, keepParams['keep-yearly'], 'keep-yearly', function(backup) {
+		    return Ext.Date.format(backup.backuptime, 'Y');
+		});
+
+		backups.forEach(function(backup) {
+		    backup.mark = backup.mark || 'remove';
+		});
+	    },
+
+	    toggleColors: function(checkbox, checked) {
+		this.switchColor(checked);
+	    },
+
+	    switchColor: function(useColors) {
+		let me = this;
+		let view = me.getView();
+
+		const getStyle = name =>
+		    `background-color: ${COLORS[name]}; color: ${TEXT_COLORS[name]};`;
+
+		for (const field of view.query('[isFormField]')) {
+		    if (field.fieldGroup !== 'keep') {
+			continue;
+		    }
+		    if (useColors) {
+		    field.setFieldStyle(getStyle(field.name));
+		    } else {
+			field.setFieldStyle('background-color: white; color: #444;');
+		    }
+		}
+
+		me.lookup('weekTable').useColors = useColors;
+		me.lookup('pruneList').useColors = useColors;
+
+		me.reloadPrune();
+	    },
+	},
+
+	keepItems: [
+	    {
+		xtype: 'numberfield',
+		name: 'keep-last',
+		allowBlank: true,
+		fieldLabel: 'keep-last',
+		minValue: 0,
+		value: 4,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-hourly',
+		allowBlank: true,
+		fieldLabel: 'keep-hourly',
+		minValue: 0,
+		value: 0,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-daily',
+		allowBlank: true,
+		fieldLabel: 'keep-daily',
+		minValue: 0,
+		value: 5,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-weekly',
+		allowBlank: true,
+		fieldLabel: 'keep-weekly',
+		minValue: 0,
+		value: 2,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-monthly',
+		allowBlank: true,
+		fieldLabel: 'keep-monthly',
+		minValue: 0,
+		value: 0,
+		fieldGroup: 'keep',
+	    },
+	    {
+		xtype: 'numberfield',
+		name: 'keep-yearly',
+		allowBlank: true,
+		fieldLabel: 'keep-yearly',
+		minValue: 0,
+		value: 0,
+		fieldGroup: 'keep',
+	    },
+	],
+
+	initComponent: function() {
+	    var me = this;
+
+	    me.pruneStore = Ext.create('Ext.data.Store', {
+		model: 'pbs-prune-list',
+		sorters: { property: 'backuptime', direction: 'DESC' },
+	    });
+
+	    let scheduleItems = [
+		{
+		    xtype: 'prunesimulatorDayOfWeekSelector',
+		    name: 'schedule-weekdays',
+		    fieldLabel: 'Day of week',
+		    value: ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'],
+		    allowBlank: false,
+		    multiSelect: true,
+		    padding: '0 0 0 10',
+		},
+		{
+		    xtype: 'prunesimulatorCalendarEvent',
+		    name: 'schedule-time',
+		    allowBlank: false,
+		    value: '0/6:00',
+		    fieldLabel: 'Backup schedule',
+		    padding: '0 0 0 10',
+		},
+		{
+		    xtype: 'numberfield',
+		    name: 'numberOfWeeks',
+		    allowBlank: false,
+		    fieldLabel: 'Number of weeks',
+		    minValue: 1,
+		    value: 15,
+		    maxValue: 260, // five years
+		    padding: '0 0 0 10',
+		},
+		{
+		    xtype: 'button',
+		    name: 'schedule-button',
+		    text: 'Update Schedule',
+		    handler: 'reloadFull',
+		},
+	    ];
+
+	    me.items = [
+		{
+		    xtype: 'panel',
+		    layout: {
+			type: 'hbox',
+			align: 'stretch',
+		    },
+		    border: false,
+		    items: [
+			{
+			    title: 'View',
+			    layout: 'anchor',
+			    flex: 1,
+			    border: false,
+			    bodyPadding: 10,
+			    items: [
+				{
+				    xtype: 'checkbox',
+				    name: 'showCalendar',
+				    reference: 'showCalendar',
+				    fieldLabel: 'Show Calendar:',
+				    checked: true,
+				},
+				{
+				    xtype: 'checkbox',
+				    name: 'showColors',
+				    reference: 'showColors',
+				    fieldLabel: 'Show Colors:',
+				    checked: true,
+				    handler: 'toggleColors',
+				},
+			    ],
+			},
+			{ xtype: "panel", width: 1, border: 1 },
+			{
+			    layout: 'anchor',
+			    flex: 1,
+			    border: false,
+			    title: 'Simulated Backup Schedule',
+			    defaults: {
+				labelWidth: 120,
+			    },
+			    bodyPadding: 10,
+			    items: scheduleItems,
+			},
+		    ],
+		},
+		{
+		    xtype: 'panel',
+		    layout: {
+			type: 'hbox',
+			align: 'stretch',
+		    },
+		    flex: 1,
+		    border: false,
+		    items: [
+			{
+			    layout: 'anchor',
+			    title: 'Prune Options',
+			    border: false,
+			    bodyPadding: 10,
+			    scrollable: true,
+			    items: me.keepItems,
+			    flex: 1,
+			},
+			{ xtype: "panel", width: 1, border: 1 },
+			{
+			    layout: 'fit',
+			    title: 'Backups',
+			    border: false,
+			    xtype: 'prunesimulatorPruneList',
+			    store: me.pruneStore,
+			    reference: 'pruneList',
+			    flex: 1,
+			},
+		    ],
+		},
+		{
+		    layout: 'anchor',
+		    title: 'Calendar',
+		    autoScroll: true,
+		    flex: 2,
+		    xtype: 'prunesimulatorWeekTable',
+		    reference: 'weekTable',
+		    store: me.pruneStore,
+		    bind: {
+			hidden: '{!showCalendar.checked}',
+		    },
+		},
+	    ];
+
+	    me.callParent();
+	},
+    });
+
+    Ext.create('Ext.container.Viewport', {
+	layout: 'border',
+	renderTo: Ext.getBody(),
+	items: [
+	    {
+		xtype: 'prunesimulatorPanel',
+		title: 'Proxmox Backup Server - Prune Simulator',
+		region: 'west',
+		layout: {
+		    type: 'vbox',
+		    align: 'stretch',
+		    pack: 'start',
+		},
+		flex: 3,
+		maxWidth: 1090,
+	    },
+	    {
+		xtype: 'prunesimulatorDocumentation',
+		title: 'Usage',
+		border: false,
+		flex: 2,
+		region: 'center',
+	    },
+	],
+    });
+});
+

docs/storage.rst

@@ -107,7 +107,7 @@ is stored in the file ``/etc/proxmox-backup/datastore.cfg``.
 Datastore Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-.. image:: images/screenshots/pbs-gui-datastore.png
+.. image:: images/screenshots/pbs-gui-datastore-content.png
   :align: right
   :alt: Datastore Overview
 
@@ -127,8 +127,9 @@ Creating a Datastore
   :align: right
   :alt: Create a datastore
 
-You can create a new datastore from the web GUI, by navigating to **Datastore** in
-the menu tree and clicking **Create**. Here:
+You can create a new datastore from the web interface, by clicking **Add
+Datastore** in the side menu, under the **Datastore** section. In the setup
+window:
 
 * *Name* refers to the name of the datastore
 * *Backing Path* is the path to the directory upon which you want to create the
@@ -136,7 +137,9 @@ the menu tree and clicking **Create**. Here:
 * *GC Schedule* refers to the time and intervals at which garbage collection
   runs
 * *Prune Schedule* refers to the frequency at which pruning takes place
-* *Prune Options* set the amount of backups which you would like to keep (see :ref:`backup-pruning`).
+* *Prune Options* set the amount of backups which you would like to keep (see
+  :ref:`backup-pruning`).
+* *Comment* can be used to add some contextual information to the datastore.
 
 Alternatively you can create a new datastore from the command line. The
 following command creates a new datastore called ``store1`` on :file:`/backup/disk1/store1`

docs/user-management.rst

@@ -41,11 +41,12 @@ users:
   :alt: Add a new user
 
 The superuser has full administration rights on everything, so you
-normally want to add other users with less privileges. You can create a new
-user with the ``user create`` subcommand or through the web interface, under
-**Configuration -> User Management**. The ``create`` subcommand lets you specify
-many options like ``--email`` or ``--password``. You can update or change any
-user properties using the ``update`` subcommand later (**Edit** in the GUI):
+normally want to add other users with less privileges. You can add a new
+user with the ``user create`` subcommand or through the web
+interface, under the **User Management** tab of **Configuration -> Access
+Control**. The ``create`` subcommand lets you specify many options like
+``--email`` or ``--password``. You can update or change any user properties
+using the ``update`` subcommand later (**Edit** in the GUI):
 
 
 .. code-block:: console
@@ -70,7 +71,7 @@ The resulting user list looks like this:
   │ root@pam │      1 │        │           │          │                  │ Superuser        │
   └──────────┴────────┴────────┴───────────┴──────────┴──────────────────┴──────────────────┘
 
-Newly created users do not have any permissions. Please read the next
+Newly created users do not have any permissions. Please read the Access Control
 section to learn how to set access permissions.
 
 If you want to disable a user account, you can do that by setting ``--enable`` to ``0``
@@ -85,15 +86,77 @@ Or completely remove the user with:
 
   # proxmox-backup-manager user remove john@pbs
 
+.. _user_tokens:
+
+API Tokens
+----------
+
+.. image:: images/screenshots/pbs-gui-apitoken-overview.png
+  :align: right
+  :alt: API Token Overview
+
+Any authenticated user can generate API tokens which can in turn be used to
+configure various clients, instead of directly providing the username and
+password.
+
+API tokens serve two purposes:
+
+#. Easy revocation in case client gets compromised
+#. Limit permissions for each client/token within the users' permission
+
+An API token consists of two parts: an identifier consisting of the user name,
+the realm and a tokenname (``user@realm!tokenname``), and a secret value. Both
+need to be provided to the client in place of the user ID (``user@realm``) and
+the user password, respectively.
+
+.. image:: images/screenshots/pbs-gui-apitoken-secret-value.png
+  :align: right
+  :alt: API secret value
+
+The API token is passed from the client to the server by setting the
+``Authorization`` HTTP header with method ``PBSAPIToken`` to the value
+``TOKENID:TOKENSECRET``.
+
+Generating new tokens can done using ``proxmox-backup-manager`` or the GUI:
+
+.. code-block:: console
+
+  # proxmox-backup-manager user generate-token john@pbs client1
+  Result: {
+    "tokenid": "john@pbs!client1",
+    "value": "d63e505a-e3ec-449a-9bc7-1da610d4ccde"
+  }
+
+.. note:: The displayed secret value needs to be saved, since it cannot be
+  displayed again after generating the API token.
+
+The ``user list-tokens`` sub-command can be used to display tokens and their
+metadata:
+
+.. code-block:: console
+
+  # proxmox-backup-manager user list-tokens john@pbs
+  ┌──────────────────┬────────┬────────┬─────────┐
+  │ tokenid          │ enable │ expire │ comment │
+  ╞══════════════════╪════════╪════════╪═════════╡
+  │ john@pbs!client1 │      1 │        │         │
+  └──────────────────┴────────┴────────┴─────────┘
+
+Similarly, the ``user delete-token`` subcommand can be used to delete a token
+again.
+
+Newly generated API tokens don't have any permissions. Please read the next
+section to learn how to set access permissions.
+
 
 .. _user_acl:
 
 Access Control
 --------------
 
-By default new users do not have any permission. Instead you need to
-specify what is allowed and what is not. You can do this by assigning
-roles to users on specific objects like datastores or remotes. The
+By default new users and API tokens do not have any permission. Instead you
+need to specify what is allowed and what is not. You can do this by assigning
+roles to users/tokens on specific objects like datastores or remotes. The
 following roles exist:
 
 **NoAccess**
@@ -130,7 +193,7 @@ following roles exist:
 **RemoteSyncOperator**
   Is allowed to read data from a remote.
 
-.. image:: images/screenshots/pbs-gui-permissions-add.png
+.. image:: images/screenshots/pbs-gui-user-management-add-user.png
   :align: right
   :alt: Add permissions for user
 
@@ -148,20 +211,21 @@ The data represented in each field is as follows:
 #. The object on which the permission is set. This can be a specific object
    (single datastore, remote, etc.) or a top level object, which with
    propagation enabled, represents all children of the object also.
-#. The user for which the permission is set
+#. The user(s)/token(s) for which the permission is set
 #. The role being set
 
-You can manage datastore permissions from **Configuration -> Permissions** in the
-web interface. Likewise, you can use the ``acl`` subcommand to manage and
-monitor user permissions from the command line. For example, the command below
-will add the user ``john@pbs`` as a **DatastoreAdmin** for the datastore
-``store1``, located at ``/backup/disk1/store1``:
+You can manage permissions via **Configuration -> Access Control ->
+Permissions** in the web interface. Likewise, you can use the ``acl``
+subcommand to manage and monitor user permissions from the command line. For
+example, the command below will add the user ``john@pbs`` as a
+**DatastoreAdmin** for the datastore ``store1``, located at
+``/backup/disk1/store1``:
 
 .. code-block:: console
 
-  # proxmox-backup-manager acl update /datastore/store1 DatastoreAdmin --userid john@pbs
+  # proxmox-backup-manager acl update /datastore/store1 DatastoreAdmin --auth-id john@pbs
 
-You can monitor the roles of each user using the following command:
+You can list the ACLs of each user/token using the following command:
 
 .. code-block:: console
 
@@ -172,7 +236,7 @@ You can monitor the roles of each user using the following command:
    │ john@pbs │ /datastore/disk1 │         1 │ DatastoreAdmin │
    └──────────┴──────────────────┴───────────┴────────────────┘
 
-A single user can be assigned multiple permission sets for different datastores.
+A single user/token can be assigned multiple permission sets for different datastores.
 
 .. Note::
   Naming convention is important here. For datastores on the host,
@@ -183,4 +247,39 @@ A single user can be assigned multiple permission sets for different datastores.
   remote (see `Remote` below) and ``{storename}`` is the name of the datastore on
   the remote.
 
+API Token permissions
+~~~~~~~~~~~~~~~~~~~~~
 
+API token permissions are calculated based on ACLs containing their ID
+independent of those of their corresponding user. The resulting permission set
+on a given path is then intersected with that of the corresponding user.
+
+In practice this means:
+
+#. API tokens require their own ACL entries
+#. API tokens can never do more than their corresponding user
+
+Effective permissions
+~~~~~~~~~~~~~~~~~~~~~
+
+To calculate and display the effective permission set of a user or API token
+you can use the ``proxmox-backup-manager user permission`` command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager user permissions john@pbs -- path /datastore/store1
+  Privileges with (*) have the propagate flag set
+  
+  Path: /datastore/store1
+  - Datastore.Audit (*)
+  - Datastore.Backup (*)
+  - Datastore.Modify (*)
+  - Datastore.Prune (*)
+  - Datastore.Read (*)
+  
+  # proxmox-backup-manager acl update /datastore/store1 DatastoreBackup --auth-id 'john@pbs!client1'
+  # proxmox-backup-manager user permissions 'john@pbs!test' -- path /datastore/store1
+  Privileges with (*) have the propagate flag set
+  
+  Path: /datastore/store1
+  - Datastore.Backup (*)

etc/Makefile

@@ -1,9 +1,11 @@
 include ../defines.mk
 
-UNITS :=
+UNITS := \
+	proxmox-backup-daily-update.timer \
 
 DYNAMIC_UNITS := \
 	proxmox-backup-banner.service \
+	proxmox-backup-daily-update.service \
 	proxmox-backup.service \
 	proxmox-backup-proxy.service
 

etc/proxmox-backup-daily-update.service.in

@@ -0,0 +1,8 @@
+[Unit]
+Description=Daily Proxmox Backup Server update and maintenance activities
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=%LIBEXECDIR%/proxmox-backup/proxmox-daily-update

etc/proxmox-backup-daily-update.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily Proxmox Backup Server update and maintenance activities
+
+[Timer]
+OnCalendar=*-*-* 1:00
+RandomizedDelaySec=5h
+Persistent=true
+
+[Install]
+WantedBy=timers.target

etc/proxmox-backup-proxy.service.in

@@ -9,6 +9,7 @@ After=proxmox-backup.service
 Type=notify
 ExecStart=%LIBEXECDIR%/proxmox-backup/proxmox-backup-proxy
 ExecReload=/bin/kill -HUP $MAINPID
+PIDFile=/run/proxmox-backup/proxy.pid
 Restart=on-failure
 User=%PROXY_USER%
 Group=%PROXY_USER%

etc/proxmox-backup.service.in

@@ -7,6 +7,7 @@ After=network.target
 Type=notify
 ExecStart=%LIBEXECDIR%/proxmox-backup/proxmox-backup-api
 ExecReload=/bin/kill -HUP $MAINPID
+PIDFile=/run/proxmox-backup/api.pid
 Restart=on-failure
 
 [Install]

examples/download-speed.rs

@@ -2,7 +2,7 @@ use std::io::Write;
 
 use anyhow::{Error};
 
-use proxmox_backup::api2::types::Userid;
+use proxmox_backup::api2::types::Authid;
 use proxmox_backup::client::{HttpClient, HttpClientOptions, BackupReader};
 
 pub struct DummyWriter {
@@ -26,13 +26,13 @@ async fn run() -> Result<(), Error> {
 
     let host = "localhost";
 
-    let username = Userid::root_userid();
+    let auth_id = Authid::root_auth_id();
 
     let options = HttpClientOptions::new()
         .interactive(true)
         .ticket_cache(true);
 
-    let client = HttpClient::new(host, 8007, username, options)?;
+    let client = HttpClient::new(host, 8007, auth_id, options)?;
 
     let backup_time = proxmox::tools::time::parse_rfc3339("2019-06-28T10:49:48Z")?;
 

examples/upload-speed.rs

@@ -1,6 +1,6 @@
 use anyhow::{Error};
 
-use proxmox_backup::api2::types::Userid;
+use proxmox_backup::api2::types::Authid;
 use proxmox_backup::client::*;
 
 async fn upload_speed() -> Result<f64, Error> {
@@ -8,13 +8,13 @@ async fn upload_speed() -> Result<f64, Error> {
     let host = "localhost";
     let datastore = "store2";
 
-    let username = Userid::root_userid();
+    let auth_id = Authid::root_auth_id();
 
     let options = HttpClientOptions::new()
         .interactive(true)
         .ticket_cache(true);
 
-    let client = HttpClient::new(host, 8007, username, options)?;
+    let client = HttpClient::new(host, 8007, auth_id, options)?;
 
     let backup_time = proxmox::tools::time::epoch_i64();
 

src/api2.rs

@@ -7,6 +7,7 @@ pub mod reader;
 pub mod status;
 pub mod types;
 pub mod version;
+pub mod ping;
 pub mod pull;
 mod helpers;
 
@@ -22,6 +23,7 @@ pub const SUBDIRS: SubdirMap = &[
     ("backup", &backup::ROUTER),
     ("config", &config::ROUTER),
     ("nodes", &NODES_ROUTER),
+    ("ping", &ping::ROUTER),
     ("pull", &pull::ROUTER),
     ("reader", &reader::ROUTER),
     ("status", &status::ROUTER),

src/api2/access.rs

@@ -1,6 +1,8 @@
 use anyhow::{bail, format_err, Error};
 
 use serde_json::{json, Value};
+use std::collections::HashMap;
+use std::collections::HashSet;
 
 use proxmox::api::{api, RpcEnvironment, Permission};
 use proxmox::api::router::{Router, SubdirMap};
@@ -11,8 +13,9 @@ use crate::tools::ticket::{self, Empty, Ticket};
 use crate::auth_helpers::*;
 use crate::api2::types::*;
 
+use crate::config::acl as acl_config;
+use crate::config::acl::{PRIVILEGES, PRIV_SYS_AUDIT, PRIV_PERMISSIONS_MODIFY};
 use crate::config::cached_user_info::CachedUserInfo;
-use crate::config::acl::{PRIVILEGES, PRIV_PERMISSIONS_MODIFY};
 
 pub mod user;
 pub mod domain;
@@ -30,7 +33,8 @@ fn authenticate_user(
 ) -> Result<bool, Error> {
     let user_info = CachedUserInfo::new()?;
 
-    if !user_info.is_active_user(&userid) {
+    let auth_id = Authid::from(userid.clone());
+    if !user_info.is_active_auth_id(&auth_id) {
         bail!("user account disabled or expired.");
     }
 
@@ -68,8 +72,7 @@ fn authenticate_user(
                             path_vec.push(part);
                         }
                     }
-
-                    user_info.check_privs(userid, &path_vec, *privilege, false)?;
+                    user_info.check_privs(&auth_id, &path_vec, *privilege, false)?;
                     return Ok(false);
                 }
             }
@@ -138,6 +141,7 @@ fn create_ticket(
     path: Option<String>,
     privs: Option<String>,
     port: Option<u16>,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
     match authenticate_user(&username, &password, path, privs, port) {
         Ok(true) => {
@@ -145,7 +149,7 @@ fn create_ticket(
 
             let token = assemble_csrf_prevention_token(csrf_secret(), &username);
 
-            log::info!("successful auth for user '{}'", username);
+            crate::server::rest::auth_logger()?.log(format!("successful auth for user '{}'", username));
 
             Ok(json!({
                 "username": username,
@@ -157,8 +161,20 @@ fn create_ticket(
             "username": username,
         })),
         Err(err) => {
-            let client_ip = "unknown"; // $rpcenv->get_client_ip() || '';
-            log::error!("authentication failure; rhost={} user={} msg={}", client_ip, username, err.to_string());
+            let client_ip = match rpcenv.get_client_ip().map(|addr| addr.ip()) {
+                Some(ip) => format!("{}", ip),
+                None => "unknown".into(),
+            };
+
+            let msg = format!(
+                "authentication failure; rhost={} user={} msg={}",
+                client_ip,
+                username,
+                err.to_string()
+            );
+            crate::server::rest::auth_logger()?.log(&msg);
+            log::error!("{}", msg);
+
             Err(http_err!(UNAUTHORIZED, "permission check failed."))
         }
     }
@@ -192,9 +208,10 @@ fn change_password(
 ) -> Result<Value, Error> {
 
     let current_user: Userid = rpcenv
-        .get_user()
+        .get_auth_id()
         .ok_or_else(|| format_err!("unknown user"))?
         .parse()?;
+    let current_auth = Authid::from(current_user.clone());
 
     let mut allowed = userid == current_user;
 
@@ -202,7 +219,7 @@ fn change_password(
 
     if !allowed {
         let user_info = CachedUserInfo::new()?;
-        let privs = user_info.lookup_privs(&current_user, &[]);
+        let privs = user_info.lookup_privs(&current_auth, &[]);
         if (privs & PRIV_PERMISSIONS_MODIFY) != 0 { allowed = true; }
     }
 
@@ -216,6 +233,128 @@ fn change_password(
     Ok(Value::Null)
 }
 
+#[api(
+    input: {
+        properties: {
+            "auth-id": {
+                type: Authid,
+                optional: true,
+            },
+            path: {
+                schema: ACL_PATH_SCHEMA,
+                optional: true,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Sys.Audit on '/access', limited to own privileges otherwise.",
+    },
+    returns: {
+        description: "Map of ACL path to Map of privilege to propagate bit",
+        type: Object,
+        properties: {},
+        additional_properties: true,
+    },
+)]
+/// List permissions of given or currently authenticated user / API token.
+///
+/// Optionally limited to specific path.
+pub fn list_permissions(
+    auth_id: Option<Authid>,
+    path: Option<String>,
+    rpcenv: &dyn RpcEnvironment,
+) -> Result<HashMap<String, HashMap<String, bool>>, Error> {
+    let current_auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+
+    let user_info = CachedUserInfo::new()?;
+    let user_privs = user_info.lookup_privs(&current_auth_id, &["access"]);
+
+    let auth_id = if user_privs & PRIV_SYS_AUDIT == 0 {
+        match auth_id {
+            Some(auth_id) => {
+                if auth_id == current_auth_id {
+                    auth_id
+                } else if auth_id.is_token()
+                    && !current_auth_id.is_token()
+                    && auth_id.user() == current_auth_id.user() {
+                    auth_id
+                } else {
+                    bail!("not allowed to list permissions of {}", auth_id);
+                }
+            },
+            None => current_auth_id,
+        }
+    } else {
+        match auth_id {
+            Some(auth_id) => auth_id,
+            None => current_auth_id,
+        }
+    };
+
+
+    fn populate_acl_paths(
+        mut paths: HashSet<String>,
+        node: acl_config::AclTreeNode,
+        path: &str
+    ) -> HashSet<String> {
+        for (sub_path, child_node) in node.children {
+            let sub_path = format!("{}/{}", path, &sub_path);
+            paths = populate_acl_paths(paths, child_node, &sub_path);
+            paths.insert(sub_path);
+        }
+        paths
+    }
+
+    let paths = match path {
+        Some(path) => {
+            let mut paths = HashSet::new();
+            paths.insert(path);
+            paths
+        },
+        None => {
+            let mut paths = HashSet::new();
+
+            let (acl_tree, _) = acl_config::config()?;
+            paths = populate_acl_paths(paths, acl_tree.root, "");
+
+            // default paths, returned even if no ACL exists
+            paths.insert("/".to_string());
+            paths.insert("/access".to_string());
+            paths.insert("/datastore".to_string());
+            paths.insert("/remote".to_string());
+            paths.insert("/system".to_string());
+
+            paths
+        },
+    };
+
+    let map = paths
+        .into_iter()
+        .fold(HashMap::new(), |mut map: HashMap<String, HashMap<String, bool>>, path: String| {
+            let split_path = acl_config::split_acl_path(path.as_str());
+            let (privs, propagated_privs) = user_info.lookup_privs_details(&auth_id, &split_path);
+
+            match privs {
+                0 => map, // Don't leak ACL paths where we don't have any privileges
+                _ => {
+                    let priv_map = PRIVILEGES
+                        .iter()
+                        .fold(HashMap::new(), |mut priv_map, (name, value)| {
+                            if value & privs != 0 {
+                                priv_map.insert(name.to_string(), value & propagated_privs != 0);
+                            }
+                            priv_map
+                        });
+
+                    map.insert(path, priv_map);
+                    map
+                },
+            }});
+
+    Ok(map)
+}
+
 #[sortable]
 const SUBDIRS: SubdirMap = &sorted!([
     ("acl", &acl::ROUTER),
@@ -223,6 +362,10 @@ const SUBDIRS: SubdirMap = &sorted!([
         "password", &Router::new()
             .put(&API_METHOD_CHANGE_PASSWORD)
     ),
+    (
+        "permissions", &Router::new()
+            .get(&API_METHOD_LIST_PERMISSIONS)
+    ),
     (
         "ticket", &Router::new()
             .post(&API_METHOD_CREATE_TICKET)

src/api2/access/acl.rs

@@ -7,6 +7,7 @@ use proxmox::tools::fs::open_file_locked;
 use crate::api2::types::*;
 use crate::config::acl;
 use crate::config::acl::{Role, PRIV_SYS_AUDIT, PRIV_PERMISSIONS_MODIFY};
+use crate::config::cached_user_info::CachedUserInfo;
 
 #[api(
     properties: {
@@ -43,8 +44,23 @@ fn extract_acl_node_data(
     path: &str,
     list: &mut Vec<AclListItem>,
     exact: bool,
+    token_user: &Option<Authid>,
 ) {
+    // tokens can't have tokens, so we can early return
+    if let Some(token_user) = token_user {
+        if token_user.is_token() {
+            return;
+        }
+    }
+
     for (user, roles) in &node.users {
+        if let Some(token_user) = token_user {
+            if !user.is_token()
+                || user.user() != token_user.user() {
+                 continue;
+            }
+        }
+
         for (role, propagate) in roles {
             list.push(AclListItem {
                 path: if path.is_empty() { String::from("/") } else { path.to_string() },
@@ -56,6 +72,10 @@ fn extract_acl_node_data(
         }
     }
     for (group, roles) in &node.groups {
+        if let Some(_) = token_user {
+            continue;
+        }
+
         for (role, propagate) in roles {
             list.push(AclListItem {
                 path: if path.is_empty() { String::from("/") } else { path.to_string() },
@@ -71,7 +91,7 @@ fn extract_acl_node_data(
     }
     for (comp, child) in &node.children {
         let new_path = format!("{}/{}", path, comp);
-        extract_acl_node_data(child, &new_path, list, exact);
+        extract_acl_node_data(child, &new_path, list, exact, token_user);
     }
 }
 
@@ -98,7 +118,8 @@ fn extract_acl_node_data(
         }
     },
     access: {
-        permission: &Permission::Privilege(&["access", "acl"], PRIV_SYS_AUDIT, false),
+        permission: &Permission::Anybody,
+        description: "Returns all ACLs if user has Sys.Audit on '/access/acl', or just the ACLs containing the user's API tokens.",
     },
 )]
 /// Read Access Control List (ACLs).
@@ -107,18 +128,26 @@ pub fn read_acl(
     exact: bool,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<AclListItem>, Error> {
+    let auth_id = rpcenv.get_auth_id().unwrap().parse()?;
 
-    //let auth_user = rpcenv.get_user().unwrap();
+    let user_info = CachedUserInfo::new()?;
+
+    let top_level_privs = user_info.lookup_privs(&auth_id, &["access", "acl"]);
+    let auth_id_filter = if (top_level_privs & PRIV_SYS_AUDIT) == 0 {
+        Some(auth_id)
+    } else {
+        None
+    };
 
     let (mut tree, digest) = acl::config()?;
 
     let mut list: Vec<AclListItem> = Vec::new();
     if let Some(path) = &path {
         if let Some(node) = &tree.find_node(path) {
-            extract_acl_node_data(&node, path, &mut list, exact);
+            extract_acl_node_data(&node, path, &mut list, exact, &auth_id_filter);
         }
     } else {
-        extract_acl_node_data(&tree.root, "", &mut list, exact);
+        extract_acl_node_data(&tree.root, "", &mut list, exact, &auth_id_filter);
     }
 
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
@@ -140,9 +169,9 @@ pub fn read_acl(
                 optional: true,
                 schema: ACL_PROPAGATE_SCHEMA,
             },
-            userid: {
+            "auth-id": {
                 optional: true,
-                type: Userid,
+                type: Authid,
             },
             group: {
                 optional: true,
@@ -160,7 +189,8 @@ pub fn read_acl(
        },
     },
     access: {
-        permission: &Permission::Privilege(&["access", "acl"], PRIV_PERMISSIONS_MODIFY, false),
+        permission: &Permission::Anybody,
+        description: "Requires Permissions.Modify on '/access/acl', limited to updating ACLs of the user's API tokens otherwise."
     },
 )]
 /// Update Access Control List (ACLs).
@@ -168,12 +198,35 @@ pub fn update_acl(
     path: String,
     role: String,
     propagate: Option<bool>,
-    userid: Option<Userid>,
+    auth_id: Option<Authid>,
     group: Option<String>,
     delete: Option<bool>,
     digest: Option<String>,
-    _rpcenv: &mut dyn RpcEnvironment,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<(), Error> {
+    let current_auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+
+    let user_info = CachedUserInfo::new()?;
+
+    let top_level_privs = user_info.lookup_privs(&current_auth_id, &["access", "acl"]);
+    if top_level_privs & PRIV_PERMISSIONS_MODIFY == 0 {
+        if let Some(_) = group {
+            bail!("Unprivileged users are not allowed to create group ACL item.");
+        }
+
+        match &auth_id {
+            Some(auth_id) => {
+                if current_auth_id.is_token() {
+                    bail!("Unprivileged API tokens can't set ACL items.");
+                } else if !auth_id.is_token() {
+                    bail!("Unprivileged users can only set ACL items for API tokens.");
+                } else if auth_id.user() != current_auth_id.user() {
+                    bail!("Unprivileged users can only set ACL items for their own API tokens.");
+                }
+            },
+            None => { bail!("Unprivileged user needs to provide auth_id to update ACL item."); },
+        };
+    }
 
     let _lock = open_file_locked(acl::ACL_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
@@ -190,11 +243,12 @@ pub fn update_acl(
 
     if let Some(ref _group) = group {
         bail!("parameter 'group' - groups are currently not supported.");
-    } else if let Some(ref userid) = userid {
+    } else if let Some(ref auth_id) = auth_id {
         if !delete { // Note: we allow to delete non-existent users
             let user_cfg = crate::config::user::cached_config()?;
-            if user_cfg.sections.get(&userid.to_string()).is_none() {
-                bail!("no such user.");
+            if user_cfg.sections.get(&auth_id.to_string()).is_none() {
+                bail!(format!("no such {}.",
+                              if auth_id.is_token() { "API token" } else { "user" }));
             }
         }
     } else {
@@ -205,11 +259,11 @@ pub fn update_acl(
         acl::check_acl_path(&path)?;
     }
 
-    if let Some(userid) = userid {
+    if let Some(auth_id) = auth_id {
         if delete {
-            tree.delete_user_role(&path, &userid, &role);
+            tree.delete_user_role(&path, &auth_id, &role);
         } else {
-            tree.insert_user_role(&path, &userid, &role, propagate);
+            tree.insert_user_role(&path, &auth_id, &role, propagate);
         }
     } else if let Some(group) = group {
         if delete {

src/api2/access/user.rs

@@ -1,12 +1,16 @@
 use anyhow::{bail, Error};
-use serde_json::Value;
+use serde::{Serialize, Deserialize};
+use serde_json::{json, Value};
+use std::collections::HashMap;
 
 use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
+use proxmox::api::router::SubdirMap;
 use proxmox::api::schema::{Schema, StringSchema};
 use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
 use crate::config::user;
+use crate::config::token_shadow;
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_PERMISSIONS_MODIFY};
 use crate::config::cached_user_info::CachedUserInfo;
 
@@ -16,14 +20,96 @@ pub const PBS_PASSWORD_SCHEMA: Schema = StringSchema::new("User Password.")
     .max_length(64)
     .schema();
 
+#[api(
+    properties: {
+        userid: {
+            type: Userid,
+        },
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+        enable: {
+            optional: true,
+            schema: user::ENABLE_USER_SCHEMA,
+        },
+        expire: {
+            optional: true,
+            schema: user::EXPIRE_USER_SCHEMA,
+        },
+        firstname: {
+            optional: true,
+            schema: user::FIRST_NAME_SCHEMA,
+        },
+        lastname: {
+            schema: user::LAST_NAME_SCHEMA,
+            optional: true,
+         },
+        email: {
+            schema: user::EMAIL_SCHEMA,
+            optional: true,
+        },
+        tokens: {
+            type: Array,
+            optional: true,
+            description: "List of user's API tokens.",
+            items: {
+                type: user::ApiToken
+            },
+        },
+    }
+)]
+#[derive(Serialize,Deserialize)]
+/// User properties with added list of ApiTokens
+pub struct UserWithTokens {
+    pub userid: Userid,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub comment: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub enable: Option<bool>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub expire: Option<i64>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub firstname: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub lastname: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub email: Option<String>,
+    #[serde(skip_serializing_if="Vec::is_empty", default)]
+    pub tokens: Vec<user::ApiToken>,
+}
+
+impl UserWithTokens {
+    fn new(user: user::User) -> Self {
+        Self {
+            userid: user.userid,
+            comment: user.comment,
+            enable: user.enable,
+            expire: user.expire,
+            firstname: user.firstname,
+            lastname: user.lastname,
+            email: user.email,
+            tokens: Vec::new(),
+        }
+    }
+}
+
+
 #[api(
     input: {
-        properties: {},
+        properties: {
+            include_tokens: {
+                type: bool,
+                description: "Include user's API tokens in returned list.",
+                optional: true,
+                default: false,
+            },
+        },
     },
     returns: {
         description: "List users (with config digest).",
         type: Array,
-        items: { type: user::User },
+        items: { type: UserWithTokens },
     },
     access: {
         permission: &Permission::Anybody,
@@ -32,28 +118,60 @@ pub const PBS_PASSWORD_SCHEMA: Schema = StringSchema::new("User Password.")
 )]
 /// List users
 pub fn list_users(
-    _param: Value,
+    include_tokens: bool,
     _info: &ApiMethod,
     mut rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Vec<user::User>, Error> {
+) -> Result<Vec<UserWithTokens>, Error> {
 
     let (config, digest) = user::config()?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    // intentionally user only for now
+    let userid: Userid = rpcenv.get_auth_id().unwrap().parse()?;
+    let auth_id = Authid::from(userid.clone());
+
     let user_info = CachedUserInfo::new()?;
 
-    let top_level_privs = user_info.lookup_privs(&userid, &["access", "users"]);
+    let top_level_privs = user_info.lookup_privs(&auth_id, &["access", "users"]);
     let top_level_allowed = (top_level_privs & PRIV_SYS_AUDIT) != 0;
 
     let filter_by_privs = |user: &user::User| {
         top_level_allowed || user.userid == userid
     };
 
+
     let list:Vec<user::User> = config.convert_to_typed_array("user")?;
 
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
-    Ok(list.into_iter().filter(filter_by_privs).collect())
+    let iter = list.into_iter().filter(filter_by_privs);
+    let list = if include_tokens {
+        let tokens: Vec<user::ApiToken> = config.convert_to_typed_array("token")?;
+        let mut user_to_tokens = tokens
+            .into_iter()
+            .fold(
+                HashMap::new(),
+                |mut map: HashMap<Userid, Vec<user::ApiToken>>, token: user::ApiToken| {
+                if token.tokenid.is_token() {
+                    map
+                        .entry(token.tokenid.user().clone())
+                        .or_default()
+                        .push(token);
+                }
+                map
+            });
+        iter
+            .map(|user: user::User| {
+                let mut user = UserWithTokens::new(user);
+                user.tokens = user_to_tokens.remove(&user.userid).unwrap_or_default();
+                user
+            })
+            .collect()
+    } else {
+        iter.map(|user: user::User| UserWithTokens::new(user))
+            .collect()
+    };
+
+    Ok(list)
 }
 
 #[api(
@@ -304,12 +422,340 @@ pub fn delete_user(userid: Userid, digest: Option<String>) -> Result<(), Error>
     Ok(())
 }
 
-const ITEM_ROUTER: Router = Router::new()
+#[api(
+    input: {
+        properties: {
+            userid: {
+                type: Userid,
+            },
+            tokenname: {
+                type: Tokenname,
+            },
+        },
+    },
+    returns: {
+        description: "Get API token metadata (with config digest).",
+        type: user::ApiToken,
+    },
+    access: {
+        permission: &Permission::Or(&[
+            &Permission::Privilege(&["access", "users"], PRIV_SYS_AUDIT, false),
+            &Permission::UserParam("userid"),
+        ]),
+    },
+)]
+/// Read user's API token metadata
+pub fn read_token(
+    userid: Userid,
+    tokenname: Tokenname,
+    _info: &ApiMethod,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<user::ApiToken, Error> {
+
+    let (config, digest) = user::config()?;
+
+    let tokenid = Authid::from((userid, Some(tokenname)));
+
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+    config.lookup("token", &tokenid.to_string())
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            userid: {
+                type: Userid,
+            },
+            tokenname: {
+                type: Tokenname,
+            },
+            comment: {
+                optional: true,
+                schema: SINGLE_LINE_COMMENT_SCHEMA,
+            },
+            enable: {
+                schema: user::ENABLE_USER_SCHEMA,
+                optional: true,
+            },
+            expire: {
+                schema: user::EXPIRE_USER_SCHEMA,
+                optional: true,
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Or(&[
+            &Permission::Privilege(&["access", "users"], PRIV_PERMISSIONS_MODIFY, false),
+            &Permission::UserParam("userid"),
+        ]),
+    },
+    returns: {
+        description: "API token identifier + generated secret.",
+        properties: {
+            value: {
+                type: String,
+                description: "The API token secret",
+            },
+            tokenid: {
+                type: String,
+                description: "The API token identifier",
+            },
+        },
+    },
+)]
+/// Generate a new API token with given metadata
+pub fn generate_token(
+    userid: Userid,
+    tokenname: Tokenname,
+    comment: Option<String>,
+    enable: Option<bool>,
+    expire: Option<i64>,
+    digest: Option<String>,
+) -> Result<Value, Error> {
+
+    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
+    let (mut config, expected_digest) = user::config()?;
+
+    if let Some(ref digest) = digest {
+        let digest = proxmox::tools::hex_to_digest(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    let tokenid = Authid::from((userid.clone(), Some(tokenname.clone())));
+    let tokenid_string = tokenid.to_string();
+
+    if let Some(_) = config.sections.get(&tokenid_string) {
+        bail!("token '{}' for user '{}' already exists.", tokenname.as_str(), userid);
+    }
+
+    let secret = format!("{:x}", proxmox::tools::uuid::Uuid::generate());
+    token_shadow::set_secret(&tokenid, &secret)?;
+
+    let token = user::ApiToken {
+        tokenid: tokenid.clone(),
+        comment,
+        enable,
+        expire,
+    };
+
+    config.set_data(&tokenid_string, "token", &token)?;
+
+    user::save_config(&config)?;
+
+    Ok(json!({
+        "tokenid": tokenid_string,
+        "value": secret
+    }))
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            userid: {
+                type: Userid,
+            },
+            tokenname: {
+                type: Tokenname,
+            },
+            comment: {
+                optional: true,
+                schema: SINGLE_LINE_COMMENT_SCHEMA,
+            },
+            enable: {
+                schema: user::ENABLE_USER_SCHEMA,
+                optional: true,
+            },
+            expire: {
+                schema: user::EXPIRE_USER_SCHEMA,
+                optional: true,
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Or(&[
+            &Permission::Privilege(&["access", "users"], PRIV_PERMISSIONS_MODIFY, false),
+            &Permission::UserParam("userid"),
+        ]),
+    },
+)]
+/// Update user's API token metadata
+pub fn update_token(
+    userid: Userid,
+    tokenname: Tokenname,
+    comment: Option<String>,
+    enable: Option<bool>,
+    expire: Option<i64>,
+    digest: Option<String>,
+) -> Result<(), Error> {
+
+    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
+    let (mut config, expected_digest) = user::config()?;
+
+    if let Some(ref digest) = digest {
+        let digest = proxmox::tools::hex_to_digest(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    let tokenid = Authid::from((userid, Some(tokenname)));
+    let tokenid_string = tokenid.to_string();
+
+    let mut data: user::ApiToken = config.lookup("token", &tokenid_string)?;
+
+    if let Some(comment) = comment {
+        let comment = comment.trim().to_string();
+        if comment.is_empty() {
+            data.comment = None;
+        } else {
+            data.comment = Some(comment);
+        }
+    }
+
+    if let Some(enable) = enable {
+        data.enable = if enable { None } else { Some(false) };
+    }
+
+    if let Some(expire) = expire {
+        data.expire = if expire > 0 { Some(expire) } else { None };
+    }
+
+    config.set_data(&tokenid_string, "token", &data)?;
+
+    user::save_config(&config)?;
+
+    Ok(())
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            userid: {
+                type: Userid,
+            },
+            tokenname: {
+                type: Tokenname,
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Or(&[
+            &Permission::Privilege(&["access", "users"], PRIV_PERMISSIONS_MODIFY, false),
+            &Permission::UserParam("userid"),
+        ]),
+    },
+)]
+/// Delete a user's API token
+pub fn delete_token(
+    userid: Userid,
+    tokenname: Tokenname,
+    digest: Option<String>,
+) -> Result<(), Error> {
+
+    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
+    let (mut config, expected_digest) = user::config()?;
+
+    if let Some(ref digest) = digest {
+        let digest = proxmox::tools::hex_to_digest(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    let tokenid = Authid::from((userid.clone(), Some(tokenname.clone())));
+    let tokenid_string = tokenid.to_string();
+
+    match config.sections.get(&tokenid_string) {
+        Some(_) => { config.sections.remove(&tokenid_string); },
+        None => bail!("token '{}' of user '{}' does not exist.", tokenname.as_str(), userid),
+    }
+
+    token_shadow::delete_secret(&tokenid)?;
+
+    user::save_config(&config)?;
+
+    Ok(())
+}
+
+#[api(
+    input: {
+        properties: {
+            userid: {
+                type: Userid,
+            },
+        },
+    },
+    returns: {
+        description: "List user's API tokens (with config digest).",
+        type: Array,
+        items: { type: user::ApiToken },
+    },
+    access: {
+        permission: &Permission::Or(&[
+            &Permission::Privilege(&["access", "users"], PRIV_SYS_AUDIT, false),
+            &Permission::UserParam("userid"),
+        ]),
+    },
+)]
+/// List user's API tokens
+pub fn list_tokens(
+    userid: Userid,
+    _info: &ApiMethod,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<user::ApiToken>, Error> {
+
+    let (config, digest) = user::config()?;
+
+    let list:Vec<user::ApiToken> = config.convert_to_typed_array("token")?;
+
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+    let filter_by_owner = |token: &user::ApiToken| {
+        if token.tokenid.is_token() {
+           token.tokenid.user() == &userid
+        } else {
+            false
+        }
+    };
+
+    Ok(list.into_iter().filter(filter_by_owner).collect())
+}
+
+const TOKEN_ITEM_ROUTER: Router = Router::new()
+    .get(&API_METHOD_READ_TOKEN)
+    .put(&API_METHOD_UPDATE_TOKEN)
+    .post(&API_METHOD_GENERATE_TOKEN)
+    .delete(&API_METHOD_DELETE_TOKEN);
+
+const TOKEN_ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_TOKENS)
+    .match_all("tokenname", &TOKEN_ITEM_ROUTER);
+
+const USER_SUBDIRS: SubdirMap = &[
+    ("token", &TOKEN_ROUTER),
+];
+
+const USER_ROUTER: Router = Router::new()
     .get(&API_METHOD_READ_USER)
     .put(&API_METHOD_UPDATE_USER)
-    .delete(&API_METHOD_DELETE_USER);
+    .delete(&API_METHOD_DELETE_USER)
+    .subdirs(USER_SUBDIRS);
 
 pub const ROUTER: Router = Router::new()
     .get(&API_METHOD_LIST_USERS)
     .post(&API_METHOD_CREATE_USER)
-    .match_all("userid", &ITEM_ROUTER);
+    .match_all("userid", &USER_ROUTER);

src/api2/admin.rs

@@ -3,10 +3,12 @@ use proxmox::list_subdirs_api_method;
 
 pub mod datastore;
 pub mod sync;
+pub mod verify;
 
 const SUBDIRS: SubdirMap = &[
     ("datastore", &datastore::ROUTER),
-    ("sync", &sync::ROUTER)
+    ("sync", &sync::ROUTER),
+    ("verify", &verify::ROUTER)
 ];
 
 pub const ROUTER: Router = Router::new()

src/api2/admin/datastore.rs

@@ -2,6 +2,8 @@ use std::collections::{HashSet, HashMap};
 use std::ffi::OsStr;
 use std::os::unix::ffi::OsStrExt;
 use std::sync::{Arc, Mutex};
+use std::path::{Path, PathBuf};
+use std::pin::Pin;
 
 use anyhow::{bail, format_err, Error};
 use futures::*;
@@ -16,10 +18,9 @@ use proxmox::api::{
 use proxmox::api::router::SubdirMap;
 use proxmox::api::schema::*;
 use proxmox::tools::fs::{replace_file, CreateOptions};
-use proxmox::try_block;
 use proxmox::{http_err, identity, list_subdirs_api_method, sortable};
 
-use pxar::accessor::aio::Accessor;
+use pxar::accessor::aio::{Accessor, FileContents, FileEntry};
 use pxar::EntryKind;
 
 use crate::api2::types::*;
@@ -28,24 +29,46 @@ use crate::backup::*;
 use crate::config::datastore;
 use crate::config::cached_user_info::CachedUserInfo;
 
-use crate::server::WorkerTask;
-use crate::tools::{self, AsyncReaderStream, WrappedReaderStream};
+use crate::server::{jobstate::Job, WorkerTask};
+use crate::tools::{
+    self,
+    zip::{ZipEncoder, ZipEntry},
+    AsyncChannelWriter, AsyncReaderStream, WrappedReaderStream,
+};
+
 use crate::config::acl::{
     PRIV_DATASTORE_AUDIT,
     PRIV_DATASTORE_MODIFY,
     PRIV_DATASTORE_READ,
     PRIV_DATASTORE_PRUNE,
     PRIV_DATASTORE_BACKUP,
+    PRIV_DATASTORE_VERIFY,
 };
 
-fn check_backup_owner(
+fn check_priv_or_backup_owner(
     store: &DataStore,
     group: &BackupGroup,
-    userid: &Userid,
+    auth_id: &Authid,
+    required_privs: u64,
 ) -> Result<(), Error> {
-    let owner = store.get_owner(group)?;
-    if &owner != userid {
-        bail!("backup owner check failed ({} != {})", userid, owner);
+    let user_info = CachedUserInfo::new()?;
+    let privs = user_info.lookup_privs(&auth_id, &["datastore", store.name()]);
+
+    if privs & required_privs == 0 {
+        let owner = store.get_owner(group)?;
+        check_backup_owner(&owner, auth_id)?;
+    }
+    Ok(())
+}
+
+fn check_backup_owner(
+    owner: &Authid,
+    auth_id: &Authid,
+) -> Result<(), Error> {
+    let correct_owner = owner == auth_id
+        || (owner.is_token() && &Authid::from(owner.user().clone()) == auth_id);
+    if !correct_owner {
+        bail!("backup owner check failed ({} != {})", auth_id, owner);
     }
     Ok(())
 }
@@ -143,9 +166,9 @@ fn list_groups(
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<GroupListItem>, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+    let user_privs = user_info.lookup_privs(&auth_id, &["datastore", &store]);
 
     let datastore = DataStore::lookup_datastore(&store)?;
 
@@ -165,8 +188,8 @@ fn list_groups(
 
         let list_all = (user_privs & PRIV_DATASTORE_AUDIT) != 0;
         let owner = datastore.get_owner(group)?;
-        if !list_all {
-            if owner != userid { continue; }
+        if !list_all && check_backup_owner(&owner, &auth_id).is_err() {
+            continue;
         }
 
         let result_item = GroupListItem {
@@ -224,16 +247,12 @@ pub fn list_snapshot_files(
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<BackupContent>, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-    let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
-
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let datastore = DataStore::lookup_datastore(&store)?;
 
     let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
 
-    let allowed = (user_privs & (PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_READ)) != 0;
-    if !allowed { check_backup_owner(&datastore, snapshot.group(), &userid)?; }
+    check_priv_or_backup_owner(&datastore, snapshot.group(), &auth_id, PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_READ)?;
 
     let info = BackupInfo::new(&datastore.base_path(), snapshot)?;
 
@@ -276,16 +295,12 @@ fn delete_snapshot(
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-    let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
-
     let datastore = DataStore::lookup_datastore(&store)?;
 
-    let allowed = (user_privs & PRIV_DATASTORE_MODIFY) != 0;
-    if !allowed { check_backup_owner(&datastore, snapshot.group(), &userid)?; }
+    check_priv_or_backup_owner(&datastore, snapshot.group(), &auth_id, PRIV_DATASTORE_MODIFY)?;
 
     datastore.remove_backup_dir(&snapshot, false)?;
 
@@ -332,9 +347,9 @@ pub fn list_snapshots (
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<SnapshotListItem>, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+    let user_privs = user_info.lookup_privs(&auth_id, &["datastore", &store]);
 
     let datastore = DataStore::lookup_datastore(&store)?;
 
@@ -356,8 +371,8 @@ pub fn list_snapshots (
         let list_all = (user_privs & PRIV_DATASTORE_AUDIT) != 0;
         let owner = datastore.get_owner(group)?;
 
-        if !list_all {
-            if owner != userid { continue; }
+        if !list_all && check_backup_owner(&owner, &auth_id).is_err() {
+            continue;
         }
 
         let mut size = None;
@@ -417,6 +432,53 @@ pub fn list_snapshots (
     Ok(snapshots)
 }
 
+fn get_snapshots_count(store: &DataStore) -> Result<Counts, Error> {
+    let base_path = store.base_path();
+    let backup_list = BackupInfo::list_backups(&base_path)?;
+    let mut groups = HashSet::new();
+
+    let mut result = Counts {
+        ct: None,
+        host: None,
+        vm: None,
+        other: None,
+    };
+
+    for info in backup_list {
+        let group = info.backup_dir.group();
+
+        let id = group.backup_id();
+        let backup_type = group.backup_type();
+
+        let mut new_id = false;
+
+        if groups.insert(format!("{}-{}", &backup_type, &id)) {
+            new_id = true;
+        }
+
+        let mut counts = match backup_type {
+            "ct" => result.ct.take().unwrap_or(Default::default()),
+            "host" => result.host.take().unwrap_or(Default::default()),
+            "vm" => result.vm.take().unwrap_or(Default::default()),
+            _ => result.other.take().unwrap_or(Default::default()),
+        };
+
+        counts.snapshots += 1;
+        if new_id {
+            counts.groups +=1;
+        }
+
+        match backup_type {
+            "ct" => result.ct = Some(counts),
+            "host" => result.host = Some(counts),
+            "vm" => result.vm = Some(counts),
+            _ => result.other = Some(counts),
+        }
+    }
+
+    Ok(result)
+}
+
 #[api(
     input: {
         properties: {
@@ -426,7 +488,7 @@ pub fn list_snapshots (
         },
     },
     returns: {
-        type: StorageStatus,
+        type: DataStoreStatus,
     },
     access: {
         permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_BACKUP, true),
@@ -437,9 +499,19 @@ pub fn status(
     store: String,
     _info: &ApiMethod,
     _rpcenv: &mut dyn RpcEnvironment,
-) -> Result<StorageStatus, Error> {
+) -> Result<DataStoreStatus, Error> {
     let datastore = DataStore::lookup_datastore(&store)?;
-    crate::tools::disks::disk_usage(&datastore.base_path())
+    let storage = crate::tools::disks::disk_usage(&datastore.base_path())?;
+    let counts = get_snapshots_count(&datastore)?;
+    let gc_status = datastore.last_gc_status();
+
+    Ok(DataStoreStatus {
+        total: storage.total,
+        used: storage.used,
+        avail: storage.avail,
+        gc_status,
+        counts,
+    })
 }
 
 #[api(
@@ -466,7 +538,7 @@ pub fn status(
         schema: UPID_SCHEMA,
     },
     access: {
-        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ | PRIV_DATASTORE_BACKUP, true), // fixme
+        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_VERIFY | PRIV_DATASTORE_BACKUP, true),
     },
 )]
 /// Verify backups.
@@ -482,21 +554,31 @@ pub fn verify(
 ) -> Result<Value, Error> {
     let datastore = DataStore::lookup_datastore(&store)?;
 
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let worker_id;
 
     let mut backup_dir = None;
     let mut backup_group = None;
+    let mut worker_type = "verify";
 
     match (backup_type, backup_id, backup_time) {
         (Some(backup_type), Some(backup_id), Some(backup_time)) => {
-            worker_id = format!("{}_{}_{}_{:08X}", store, backup_type, backup_id, backup_time);
+            worker_id = format!("{}:{}/{}/{:08X}", store, backup_type, backup_id, backup_time);
             let dir = BackupDir::new(backup_type, backup_id, backup_time)?;
+
+            check_priv_or_backup_owner(&datastore, dir.group(), &auth_id, PRIV_DATASTORE_VERIFY)?;
+
             backup_dir = Some(dir);
+            worker_type = "verify_snapshot";
         }
         (Some(backup_type), Some(backup_id), None) => {
-            worker_id = format!("{}_{}_{}", store, backup_type, backup_id);
+            worker_id = format!("{}:{}/{}", store, backup_type, backup_id);
             let group = BackupGroup::new(backup_type, backup_id);
+
+            check_priv_or_backup_owner(&datastore, &group, &auth_id, PRIV_DATASTORE_VERIFY)?;
+
             backup_group = Some(group);
+            worker_type = "verify_group";
         }
         (None, None, None) => {
             worker_id = store.clone();
@@ -504,13 +586,12 @@ pub fn verify(
         _ => bail!("parameters do not specify a backup group or snapshot"),
     }
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
 
     let upid_str = WorkerTask::new_thread(
-        "verify",
+        worker_type,
         Some(worker_id.clone()),
-        userid,
+        auth_id.clone(),
         to_stdout,
         move |worker| {
             let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*16)));
@@ -525,6 +606,7 @@ pub fn verify(
                     corrupt_chunks,
                     worker.clone(),
                     worker.upid().clone(),
+                    None,
                 )? {
                     res.push(backup_dir.to_string());
                 }
@@ -538,10 +620,20 @@ pub fn verify(
                     None,
                     worker.clone(),
                     worker.upid(),
+                    None,
                 )?;
                 failed_dirs
             } else {
-                verify_all_backups(datastore, worker.clone(), worker.upid())?
+                let privs = CachedUserInfo::new()?
+                    .lookup_privs(&auth_id, &["datastore", &store]);
+
+                let owner = if privs & PRIV_DATASTORE_VERIFY == 0 {
+                    Some(auth_id)
+                } else {
+                    None
+                };
+
+                verify_all_backups(datastore, worker.clone(), worker.upid(), owner, None)?
             };
             if failed_dirs.len() > 0 {
                 worker.log("Failed to verify following snapshots:");
@@ -637,9 +729,7 @@ fn prune(
     let backup_type = tools::required_string_param(&param, "backup-type")?;
     let backup_id = tools::required_string_param(&param, "backup-id")?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-    let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     let dry_run = param["dry-run"].as_bool().unwrap_or(false);
 
@@ -647,8 +737,7 @@ fn prune(
 
     let datastore = DataStore::lookup_datastore(&store)?;
 
-    let allowed = (user_privs & PRIV_DATASTORE_MODIFY) != 0;
-    if !allowed { check_backup_owner(&datastore, &group, &userid)?; }
+    check_priv_or_backup_owner(&datastore, &group, &auth_id, PRIV_DATASTORE_MODIFY)?;
 
     let prune_options = PruneOptions {
         keep_last: param["keep-last"].as_u64(),
@@ -659,7 +748,7 @@ fn prune(
         keep_yearly: param["keep-yearly"].as_u64(),
     };
 
-    let worker_id = format!("{}_{}_{}", store, backup_type, backup_id);
+    let worker_id = format!("{}:{}/{}", store, backup_type, backup_id);
 
     let mut prune_result = Vec::new();
 
@@ -690,55 +779,54 @@ fn prune(
 
 
     // We use a WorkerTask just to have a task log, but run synchrounously
-    let worker = WorkerTask::new("prune", Some(worker_id), Userid::root_userid().clone(), true)?;
+    let worker = WorkerTask::new("prune", Some(worker_id), auth_id.clone(), true)?;
 
-    let result = try_block! {
-        if keep_all {
-            worker.log("No prune selection - keeping all files.");
-        } else {
-            worker.log(format!("retention options: {}", prune_options.cli_options_string()));
-            worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
-                               store, backup_type, backup_id));
-        }
+    if keep_all {
+        worker.log("No prune selection - keeping all files.");
+    } else {
+        worker.log(format!("retention options: {}", prune_options.cli_options_string()));
+        worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
+                            store, backup_type, backup_id));
+    }
 
-        for (info, mut keep) in prune_info {
-            if keep_all { keep = true; }
+    for (info, mut keep) in prune_info {
+        if keep_all { keep = true; }
 
-            let backup_time = info.backup_dir.backup_time();
-            let timestamp = info.backup_dir.backup_time_string();
-            let group = info.backup_dir.group();
+        let backup_time = info.backup_dir.backup_time();
+        let timestamp = info.backup_dir.backup_time_string();
+        let group = info.backup_dir.group();
 
 
-            let msg = format!(
-                "{}/{}/{} {}",
-                group.backup_type(),
-                group.backup_id(),
-                timestamp,
-                if keep { "keep" } else { "remove" },
-            );
+        let msg = format!(
+            "{}/{}/{} {}",
+            group.backup_type(),
+            group.backup_id(),
+            timestamp,
+            if keep { "keep" } else { "remove" },
+        );
 
-            worker.log(msg);
+        worker.log(msg);
 
-            prune_result.push(json!({
-                "backup-type": group.backup_type(),
-                "backup-id": group.backup_id(),
-                "backup-time": backup_time,
-                "keep": keep,
-            }));
+        prune_result.push(json!({
+            "backup-type": group.backup_type(),
+            "backup-id": group.backup_id(),
+            "backup-time": backup_time,
+            "keep": keep,
+        }));
 
-            if !(dry_run || keep) {
-                datastore.remove_backup_dir(&info.backup_dir, true)?;
+        if !(dry_run || keep) {
+            if let Err(err) = datastore.remove_backup_dir(&info.backup_dir, false) {
+                worker.warn(
+                    format!(
+                        "failed to remove dir {:?}: {}",
+                        info.backup_dir.relative_path(), err
+                    )
+                );
             }
         }
+    }
 
-        Ok(())
-    };
-
-    worker.log_result(&result);
-
-    if let Err(err) = result {
-        bail!("prune failed - {}", err);
-    };
+    worker.log_result(&Ok(()));
 
     Ok(json!(prune_result))
 }
@@ -766,21 +854,15 @@ fn start_garbage_collection(
 ) -> Result<Value, Error> {
 
     let datastore = DataStore::lookup_datastore(&store)?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
-    println!("Starting garbage collection on store {}", store);
+    let job =  Job::new("garbage_collection", &store)
+        .map_err(|_| format_err!("garbage collection already running"))?;
 
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
 
-    let upid_str = WorkerTask::new_thread(
-        "garbage_collection",
-        Some(store.clone()),
-        Userid::root_userid().clone(),
-        to_stdout,
-        move |worker| {
-            worker.log(format!("starting garbage collection on store {}", store));
-            datastore.garbage_collection(&*worker, worker.upid())
-        },
-    )?;
+    let upid_str = crate::server::do_garbage_collection_job(job, datastore, &auth_id, None, to_stdout)
+        .map_err(|err| format_err!("unable to start garbage collection job on datastore {} - {}", store, err))?;
 
     Ok(json!(upid_str))
 }
@@ -820,15 +902,7 @@ pub fn garbage_collection_status(
         type: Array,
         items: {
             description: "Datastore name and description.",
-            properties: {
-                store: {
-                    schema: DATASTORE_SCHEMA,
-                },
-                comment: {
-                    optional: true,
-                    schema: SINGLE_LINE_COMMENT_SCHEMA,
-                },
-            },
+            type: DataStoreListItem,
         },
     },
     access: {
@@ -840,24 +914,25 @@ fn get_datastore_list(
     _param: Value,
     _info: &ApiMethod,
     rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
+) -> Result<Vec<DataStoreListItem>, Error> {
 
     let (config, _digest) = datastore::config()?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let user_info = CachedUserInfo::new()?;
 
     let mut list = Vec::new();
 
     for (store, (_, data)) in &config.sections {
-        let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+        let user_privs = user_info.lookup_privs(&auth_id, &["datastore", &store]);
         let allowed = (user_privs & (PRIV_DATASTORE_AUDIT| PRIV_DATASTORE_BACKUP)) != 0;
         if allowed {
-            let mut entry = json!({ "store": store });
-            if let Some(comment) = data["comment"].as_str() {
-                entry["comment"] = comment.into();
-            }
-            list.push(entry);
+            list.push(
+                DataStoreListItem {
+                    store: store.clone(),
+                    comment: data["comment"].as_str().map(String::from),
+                }
+            );
         }
     }
 
@@ -895,9 +970,7 @@ fn download_file(
         let store = tools::required_string_param(&param, "store")?;
         let datastore = DataStore::lookup_datastore(store)?;
 
-        let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-        let user_info = CachedUserInfo::new()?;
-        let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+        let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
         let file_name = tools::required_string_param(&param, "file-name")?.to_owned();
 
@@ -907,8 +980,7 @@ fn download_file(
 
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
-        let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
-        if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
+        check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_READ)?;
 
         println!("Download {} from {} ({}/{})", file_name, store, backup_dir, file_name);
 
@@ -968,9 +1040,7 @@ fn download_file_decoded(
         let store = tools::required_string_param(&param, "store")?;
         let datastore = DataStore::lookup_datastore(store)?;
 
-        let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-        let user_info = CachedUserInfo::new()?;
-        let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+        let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
         let file_name = tools::required_string_param(&param, "file-name")?.to_owned();
 
@@ -980,8 +1050,7 @@ fn download_file_decoded(
 
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
-        let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
-        if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
+        check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_READ)?;
 
         let (manifest, files) = read_backup_index(&datastore, &backup_dir)?;
         for file in files {
@@ -1093,8 +1162,9 @@ fn upload_backup_log(
 
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
-        let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-        check_backup_owner(&datastore, backup_dir.group(), &userid)?;
+        let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+        let owner = datastore.get_owner(backup_dir.group())?;
+        check_backup_owner(&owner, &auth_id)?;
 
         let mut path = datastore.base_path();
         path.push(backup_dir.relative_path());
@@ -1163,14 +1233,11 @@ fn catalog(
 ) -> Result<Value, Error> {
     let datastore = DataStore::lookup_datastore(&store)?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-    let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
-    let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
-    if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
+    check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_READ)?;
 
     let file_name = CATALOG_NAME;
 
@@ -1243,6 +1310,66 @@ fn catalog(
     Ok(res.into())
 }
 
+fn recurse_files<'a, T, W>(
+    zip: &'a mut ZipEncoder<W>,
+    decoder: &'a mut Accessor<T>,
+    prefix: &'a Path,
+    file: FileEntry<T>,
+) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'a>>
+where
+    T: Clone + pxar::accessor::ReadAt + Unpin + Send + Sync + 'static,
+    W: tokio::io::AsyncWrite + Unpin + Send + 'static,
+{
+    Box::pin(async move {
+        let metadata = file.entry().metadata();
+        let path = file.entry().path().strip_prefix(&prefix)?.to_path_buf();
+
+        match file.kind() {
+            EntryKind::File { .. } => {
+                let entry = ZipEntry::new(
+                    path,
+                    metadata.stat.mtime.secs,
+                    metadata.stat.mode as u16,
+                    true,
+                );
+                zip.add_entry(entry, Some(file.contents().await?))
+                   .await
+                   .map_err(|err| format_err!("could not send file entry: {}", err))?;
+            }
+            EntryKind::Hardlink(_) => {
+                let realfile = decoder.follow_hardlink(&file).await?;
+                let entry = ZipEntry::new(
+                    path,
+                    metadata.stat.mtime.secs,
+                    metadata.stat.mode as u16,
+                    true,
+                );
+                zip.add_entry(entry, Some(realfile.contents().await?))
+                   .await
+                   .map_err(|err| format_err!("could not send file entry: {}", err))?;
+            }
+            EntryKind::Directory => {
+                let dir = file.enter_directory().await?;
+                let mut readdir = dir.read_dir();
+                let entry = ZipEntry::new(
+                    path,
+                    metadata.stat.mtime.secs,
+                    metadata.stat.mode as u16,
+                    false,
+                );
+                zip.add_entry::<FileContents<T>>(entry, None).await?;
+                while let Some(entry) = readdir.next().await {
+                    let entry = entry?.decode_entry().await?;
+                    recurse_files(zip, decoder, prefix, entry).await?;
+                }
+            }
+            _ => {} // ignore all else
+        };
+
+        Ok(())
+    })
+}
+
 #[sortable]
 pub const API_METHOD_PXAR_FILE_DOWNLOAD: ApiMethod = ApiMethod::new(
     &ApiHandler::AsyncHttp(&pxar_file_download),
@@ -1274,9 +1401,7 @@ fn pxar_file_download(
         let store = tools::required_string_param(&param, "store")?;
         let datastore = DataStore::lookup_datastore(&store)?;
 
-        let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-        let user_info = CachedUserInfo::new()?;
-        let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+        let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
         let filepath = tools::required_string_param(&param, "filepath")?.to_owned();
 
@@ -1286,8 +1411,7 @@ fn pxar_file_download(
 
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
-        let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
-        if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
+        check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_READ)?;
 
         let mut components = base64::decode(&filepath)?;
         if components.len() > 0 && components[0] == '/' as u8 {
@@ -1325,23 +1449,55 @@ fn pxar_file_download(
             .lookup(OsStr::from_bytes(file_path)).await?
             .ok_or(format_err!("error opening '{:?}'", file_path))?;
 
-        let file = match file.kind() {
-            EntryKind::File { .. } => file,
-            EntryKind::Hardlink(_) => {
-                decoder.follow_hardlink(&file).await?
-            },
-            // TODO symlink
+        let body = match file.kind() {
+            EntryKind::File { .. } => Body::wrap_stream(
+                AsyncReaderStream::new(file.contents().await?).map_err(move |err| {
+                    eprintln!("error during streaming of file '{:?}' - {}", filepath, err);
+                    err
+                }),
+            ),
+            EntryKind::Hardlink(_) => Body::wrap_stream(
+                AsyncReaderStream::new(decoder.follow_hardlink(&file).await?.contents().await?)
+                    .map_err(move |err| {
+                        eprintln!(
+                            "error during streaming of hardlink '{:?}' - {}",
+                            filepath, err
+                        );
+                        err
+                    }),
+            ),
+            EntryKind::Directory => {
+                let (sender, receiver) = tokio::sync::mpsc::channel(100);
+                let mut prefix = PathBuf::new();
+                let mut components = file.entry().path().components();
+                components.next_back(); // discar last
+                for comp in components {
+                    prefix.push(comp);
+                }
+
+                let channelwriter = AsyncChannelWriter::new(sender, 1024 * 1024);
+
+                crate::server::spawn_internal_task(async move {
+                    let mut zipencoder = ZipEncoder::new(channelwriter);
+                    let mut decoder = decoder;
+                    recurse_files(&mut zipencoder, &mut decoder, &prefix, file)
+                        .await
+                        .map_err(|err| eprintln!("error during creating of zip: {}", err))?;
+
+                    zipencoder
+                        .finish()
+                        .await
+                        .map_err(|err| eprintln!("error during finishing of zip: {}", err))
+                });
+
+                Body::wrap_stream(receiver.map_err(move |err| {
+                    eprintln!("error during streaming of zip '{:?}' - {}", filepath, err);
+                    err
+                }))
+            }
             other => bail!("cannot download file of type {:?}", other),
         };
 
-        let body = Body::wrap_stream(
-            AsyncReaderStream::new(file.contents().await?)
-                .map_err(move |err| {
-                    eprintln!("error during streaming of '{:?}' - {}", filepath, err);
-                    err
-                })
-        );
-
         // fixme: set other headers ?
         Ok(Response::builder()
            .status(StatusCode::OK)
@@ -1408,7 +1564,7 @@ fn get_rrd_stats(
         },
     },
     access: {
-        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ | PRIV_DATASTORE_BACKUP, true),
+        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_BACKUP, true),
     },
 )]
 /// Get "notes" for a specific backup
@@ -1421,18 +1577,14 @@ fn get_notes(
 ) -> Result<String, Error> {
     let datastore = DataStore::lookup_datastore(&store)?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-    let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
-
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
-    let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
-    if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
+    check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_AUDIT)?;
 
-    let manifest = datastore.load_manifest_json(&backup_dir)?;
+    let (manifest, _) = datastore.load_manifest(&backup_dir)?;
 
-    let notes = manifest["unprotected"]["notes"]
+    let notes = manifest.unprotected["notes"]
         .as_str()
         .unwrap_or("");
 
@@ -1460,7 +1612,9 @@ fn get_notes(
         },
     },
     access: {
-        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY, true),
+        permission: &Permission::Privilege(&["datastore", "{store}"],
+                                           PRIV_DATASTORE_MODIFY | PRIV_DATASTORE_BACKUP,
+                                           true),
     },
 )]
 /// Set "notes" for a specific backup
@@ -1474,26 +1628,20 @@ fn set_notes(
 ) -> Result<(), Error> {
     let datastore = DataStore::lookup_datastore(&store)?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-    let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
-
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
-    let allowed = (user_privs & PRIV_DATASTORE_READ) != 0;
-    if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
+    check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_MODIFY)?;
 
-    let mut manifest = datastore.load_manifest_json(&backup_dir)?;
-
-    manifest["unprotected"]["notes"] = notes.into();
-
-    datastore.store_manifest(&backup_dir, manifest)?;
+    datastore.update_manifest(&backup_dir,|manifest| {
+        manifest.unprotected["notes"] = notes.into();
+    }).map_err(|err| format_err!("unable to update manifest blob - {}", err))?;
 
     Ok(())
 }
 
 #[api(
-   input: {
+    input: {
         properties: {
             store: {
                 schema: DATASTORE_SCHEMA,
@@ -1505,31 +1653,83 @@ fn set_notes(
                 schema: BACKUP_ID_SCHEMA,
             },
             "new-owner": {
-                type: Userid,
+                type: Authid,
             },
         },
-   },
-   access: {
-       permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY, true),
-   },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Datastore.Modify on whole datastore, or changing ownership between user and a user's token for owned backups with Datastore.Backup"
+    },
 )]
 /// Change owner of a backup group
 fn set_backup_owner(
     store: String,
     backup_type: String,
     backup_id: String,
-    new_owner: Userid,
-    _rpcenv: &mut dyn RpcEnvironment,
+    new_owner: Authid,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<(), Error> {
 
     let datastore = DataStore::lookup_datastore(&store)?;
 
     let backup_group = BackupGroup::new(backup_type, backup_id);
 
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+
     let user_info = CachedUserInfo::new()?;
 
-    if !user_info.is_active_user(&new_owner) {
-        bail!("user '{}' is inactive or non-existent", new_owner);
+    let privs = user_info.lookup_privs(&auth_id, &["datastore", &store]);
+
+    let allowed = if (privs & PRIV_DATASTORE_MODIFY) != 0 {
+        // High-privilege user/token
+        true
+    } else if (privs & PRIV_DATASTORE_BACKUP) != 0 {
+        let owner = datastore.get_owner(&backup_group)?;
+
+        match (owner.is_token(), new_owner.is_token()) {
+            (true, true) => {
+                // API token to API token, owned by same user
+                let owner = owner.user();
+                let new_owner = new_owner.user();
+                owner == new_owner && Authid::from(owner.clone()) == auth_id
+            },
+            (true, false) => {
+                // API token to API token owner
+                Authid::from(owner.user().clone()) == auth_id
+                    && new_owner == auth_id
+            },
+            (false, true) => {
+                // API token owner to API token
+                owner == auth_id
+                    && Authid::from(new_owner.user().clone()) == auth_id
+            },
+            (false, false) => {
+                // User to User, not allowed for unprivileged users
+                false
+            },
+        }
+    } else {
+        false
+    };
+
+    if !allowed {
+        return Err(http_err!(UNAUTHORIZED,
+                  "{} does not have permission to change owner of backup group '{}' to {}",
+                  auth_id,
+                  backup_group,
+                  new_owner,
+        ));
+    }
+
+    if !user_info.is_active_auth_id(&new_owner) {
+        bail!("{} '{}' is inactive or non-existent",
+              if new_owner.is_token() {
+                  "API token".to_string()
+              } else {
+                  "user".to_string()
+              },
+              new_owner);
     }
 
     datastore.set_owner(&backup_group, &new_owner, true)?;

src/api2/admin/sync.rs

@@ -1,37 +1,66 @@
-use anyhow::{format_err, Error};
+use anyhow::{bail, format_err, Error};
 use serde_json::Value;
 
-use proxmox::api::{api, ApiMethod, Router, RpcEnvironment};
+use proxmox::api::{api, ApiMethod, Permission, Router, RpcEnvironment};
 use proxmox::api::router::SubdirMap;
 use proxmox::{list_subdirs_api_method, sortable};
 
 use crate::api2::types::*;
 use crate::api2::pull::do_sync_job;
+use crate::api2::config::sync::{check_sync_job_modify_access, check_sync_job_read_access};
+
+use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::sync::{self, SyncJobStatus, SyncJobConfig};
 use crate::server::UPID;
-use crate::config::jobstate::{Job, JobState};
+use crate::server::jobstate::{Job, JobState};
 use crate::tools::systemd::time::{
     parse_calendar_event, compute_next_event};
 
 #[api(
     input: {
-        properties: {},
+        properties: {
+            store: {
+                schema: DATASTORE_SCHEMA,
+                optional: true,
+            },
+        },
     },
     returns: {
         description: "List configured jobs and their status.",
         type: Array,
         items: { type: sync::SyncJobStatus },
     },
+    access: {
+        description: "Limited to sync jobs where user has Datastore.Audit on target datastore, and Remote.Audit on source remote.",
+        permission: &Permission::Anybody,
+    },
 )]
 /// List all sync jobs
 pub fn list_sync_jobs(
+    store: Option<String>,
     _param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<SyncJobStatus>, Error> {
 
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
     let (config, digest) = sync::config()?;
 
-    let mut list: Vec<SyncJobStatus> = config.convert_to_typed_array("sync")?;
+    let mut list: Vec<SyncJobStatus> = config
+        .convert_to_typed_array("sync")?
+        .into_iter()
+        .filter(|job: &SyncJobStatus| {
+            if let Some(store) = &store {
+                &job.store == store
+            } else {
+                true
+            }
+        })
+        .filter(|job: &SyncJobStatus| {
+            let as_config: SyncJobConfig = job.clone().into();
+            check_sync_job_read_access(&user_info, &auth_id, &as_config)
+        }).collect();
 
     for job in &mut list {
         let last_state = JobState::load("syncjob", &job.id)
@@ -74,7 +103,11 @@ pub fn list_sync_jobs(
                 schema: JOB_ID_SCHEMA,
             }
         }
-    }
+    },
+    access: {
+        description: "User needs Datastore.Backup on target datastore, and Remote.Read on source remote. Additionally, remove_vanished requires Datastore.Prune, and any owner other than the user themselves requires Datastore.Modify",
+        permission: &Permission::Anybody,
+    },
 )]
 /// Runs the sync jobs manually.
 fn run_sync_job(
@@ -82,15 +115,19 @@ fn run_sync_job(
     _info: &ApiMethod,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<String, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let (config, _digest) = sync::config()?;
     let sync_job: SyncJobConfig = config.lookup("sync", &id)?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    if !check_sync_job_modify_access(&user_info, &auth_id, &sync_job) {
+        bail!("permission check failed");
+    }
 
     let job = Job::new("syncjob", &id)?;
 
-    let upid_str = do_sync_job(job, sync_job, &userid, None)?;
+    let upid_str = do_sync_job(job, sync_job, &auth_id, None)?;
 
     Ok(upid_str)
 }

src/api2/admin/verify.rs

@@ -0,0 +1,147 @@
+use anyhow::{format_err, Error};
+
+use proxmox::api::router::SubdirMap;
+use proxmox::{list_subdirs_api_method, sortable};
+use proxmox::api::{api, ApiMethod, Permission, Router, RpcEnvironment};
+
+use crate::api2::types::*;
+use crate::server::do_verification_job;
+use crate::server::jobstate::{Job, JobState};
+use crate::config::acl::{
+    PRIV_DATASTORE_AUDIT,
+    PRIV_DATASTORE_VERIFY,
+};
+use crate::config::cached_user_info::CachedUserInfo;
+use crate::config::verify;
+use crate::config::verify::{VerificationJobConfig, VerificationJobStatus};
+use serde_json::Value;
+use crate::tools::systemd::time::{parse_calendar_event, compute_next_event};
+use crate::server::UPID;
+
+#[api(
+    input: {
+        properties: {
+            store: {
+                schema: DATASTORE_SCHEMA,
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        description: "List configured jobs and their status (filtered by access)",
+        type: Array,
+        items: { type: verify::VerificationJobStatus },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Audit or Datastore.Verify on datastore.",
+    },
+)]
+/// List all verification jobs
+pub fn list_verification_jobs(
+    store: Option<String>,
+    _param: Value,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<VerificationJobStatus>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_VERIFY;
+
+    let (config, digest) = verify::config()?;
+
+    let mut list: Vec<VerificationJobStatus> = config
+        .convert_to_typed_array("verification")?
+        .into_iter()
+        .filter(|job: &VerificationJobStatus| {
+            let privs = user_info.lookup_privs(&auth_id, &["datastore", &job.store]);
+            if privs & required_privs == 0 {
+                return false;
+            }
+
+            if let Some(store) = &store {
+                &job.store == store
+            } else {
+                true
+            }
+        }).collect();
+
+    for job in &mut list {
+        let last_state = JobState::load("verificationjob", &job.id)
+            .map_err(|err| format_err!("could not open statefile for {}: {}", &job.id, err))?;
+
+        let (upid, endtime, state, starttime) = match last_state {
+            JobState::Created { time } => (None, None, None, time),
+            JobState::Started { upid } => {
+                let parsed_upid: UPID = upid.parse()?;
+                (Some(upid), None, None, parsed_upid.starttime)
+            },
+            JobState::Finished { upid, state } => {
+                let parsed_upid: UPID = upid.parse()?;
+                (Some(upid), Some(state.endtime()), Some(state.to_string()), parsed_upid.starttime)
+            },
+        };
+
+        job.last_run_upid = upid;
+        job.last_run_state = state;
+        job.last_run_endtime = endtime;
+
+        let last = job.last_run_endtime.unwrap_or_else(|| starttime);
+
+        job.next_run = (|| -> Option<i64> {
+            let schedule = job.schedule.as_ref()?;
+            let event = parse_calendar_event(&schedule).ok()?;
+            // ignore errors
+            compute_next_event(&event, last, false).unwrap_or_else(|_| None)
+        })();
+    }
+
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+    Ok(list)
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            }
+        }
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Verify on job's datastore.",
+    },
+)]
+/// Runs a verification job manually.
+fn run_verification_job(
+    id: String,
+    _info: &ApiMethod,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let (config, _digest) = verify::config()?;
+    let verification_job: VerificationJobConfig = config.lookup("verification", &id)?;
+
+    user_info.check_privs(&auth_id, &["datastore", &verification_job.store], PRIV_DATASTORE_VERIFY, true)?;
+
+    let job = Job::new("verificationjob", &id)?;
+
+    let upid_str = do_verification_job(job, verification_job, &auth_id, None)?;
+
+    Ok(upid_str)
+}
+
+#[sortable]
+const VERIFICATION_INFO_SUBDIRS: SubdirMap = &[("run", &Router::new().post(&API_METHOD_RUN_VERIFICATION_JOB))];
+
+const VERIFICATION_INFO_ROUTER: Router = Router::new()
+    .get(&list_subdirs_api_method!(VERIFICATION_INFO_SUBDIRS))
+    .subdirs(VERIFICATION_INFO_SUBDIRS);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_VERIFICATION_JOBS)
+    .match_all("id", &VERIFICATION_INFO_ROUTER);

src/api2/backup.rs

@@ -16,7 +16,7 @@ use crate::backup::*;
 use crate::api2::types::*;
 use crate::config::acl::PRIV_DATASTORE_BACKUP;
 use crate::config::cached_user_info::CachedUserInfo;
-use crate::tools::fs::lock_dir_noblock;
+use crate::tools::fs::lock_dir_noblock_shared;
 
 mod environment;
 use environment::*;
@@ -59,12 +59,12 @@ async move {
     let debug = param["debug"].as_bool().unwrap_or(false);
     let benchmark = param["benchmark"].as_bool().unwrap_or(false);
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     let store = tools::required_string_param(&param, "store")?.to_owned();
 
     let user_info = CachedUserInfo::new()?;
-    user_info.check_privs(&userid, &["datastore", &store], PRIV_DATASTORE_BACKUP, false)?;
+    user_info.check_privs(&auth_id, &["datastore", &store], PRIV_DATASTORE_BACKUP, false)?;
 
     let datastore = DataStore::lookup_datastore(&store)?;
 
@@ -86,7 +86,7 @@ async move {
         bail!("unexpected http version '{:?}' (expected version < 2)", parts.version);
     }
 
-    let worker_id = format!("{}_{}_{}", store, backup_type, backup_id);
+    let worker_id = format!("{}:{}/{}", store, backup_type, backup_id);
 
     let env_type = rpcenv.env_type();
 
@@ -105,12 +105,15 @@ async move {
     };
 
     // lock backup group to only allow one backup per group at a time
-    let (owner, _group_guard) = datastore.create_locked_backup_group(&backup_group, &userid)?;
+    let (owner, _group_guard) = datastore.create_locked_backup_group(&backup_group, &auth_id)?;
 
     // permission check
-    if owner != userid && worker_type != "benchmark" {
+    let correct_owner = owner == auth_id
+        || (owner.is_token()
+            && Authid::from(owner.user().clone()) == auth_id);
+    if !correct_owner && worker_type != "benchmark" {
         // only the owner is allowed to create additional snapshots
-        bail!("backup owner check failed ({} != {})", userid, owner);
+        bail!("backup owner check failed ({} != {})", auth_id, owner);
     }
 
     let last_backup = {
@@ -144,18 +147,18 @@ async move {
 
         // lock last snapshot to prevent forgetting/pruning it during backup
         let full_path = datastore.snapshot_path(&last.backup_dir);
-        Some(lock_dir_noblock(&full_path, "snapshot", "base snapshot is already locked by another operation")?)
+        Some(lock_dir_noblock_shared(&full_path, "snapshot", "base snapshot is already locked by another operation")?)
     } else {
         None
     };
 
-    let (path, is_new, _snap_guard) = datastore.create_locked_backup_dir(&backup_dir)?;
+    let (path, is_new, snap_guard) = datastore.create_locked_backup_dir(&backup_dir)?;
     if !is_new { bail!("backup directory already exists."); }
 
 
-    WorkerTask::spawn(worker_type, Some(worker_id), userid.clone(), true, move |worker| {
+    WorkerTask::spawn(worker_type, Some(worker_id), auth_id.clone(), true, move |worker| {
         let mut env = BackupEnvironment::new(
-            env_type, userid, worker.clone(), datastore, backup_dir);
+            env_type, auth_id, worker.clone(), datastore, backup_dir);
 
         env.debug = debug;
         env.last_backup = last_backup;
@@ -182,8 +185,22 @@ async move {
                 http.http2_initial_connection_window_size(window_size);
                 http.http2_max_frame_size(4*1024*1024);
 
+                let env3 = env2.clone();
                 http.serve_connection(conn, service)
-                    .map_err(Error::from)
+                    .map(move |result| {
+                        match result {
+                            Err(err) => {
+                                // Avoid  Transport endpoint is not connected (os error 107)
+                                // fixme: find a better way to test for that error
+                                if err.to_string().starts_with("connection error") && env3.finished() {
+                                    Ok(())
+                                } else {
+                                    Err(Error::from(err))
+                                }
+                            }
+                            Ok(()) => Ok(()),
+                        }
+                    })
             });
         let mut abort_future = abort_future
             .map(|_| Err(format_err!("task aborted")));
@@ -191,7 +208,7 @@ async move {
         async move {
             // keep flock until task ends
             let _group_guard = _group_guard;
-            let _snap_guard = _snap_guard;
+            let snap_guard = snap_guard;
             let _last_guard = _last_guard;
 
             let res = select!{
@@ -203,20 +220,32 @@ async move {
                 tools::runtime::block_in_place(|| env.remove_backup())?;
                 return Ok(());
             }
+
+            let verify = |env: BackupEnvironment| {
+                if let Err(err) = env.verify_after_complete(snap_guard) {
+                    env.log(format!(
+                        "backup finished, but starting the requested verify task failed: {}",
+                        err
+                    ));
+                }
+            };
+
             match (res, env.ensure_finished()) {
                 (Ok(_), Ok(())) => {
                     env.log("backup finished successfully");
+                    verify(env);
                     Ok(())
                 },
                 (Err(err), Ok(())) => {
                     // ignore errors after finish
                     env.log(format!("backup had errors but finished: {}", err));
+                    verify(env);
                     Ok(())
                 },
                 (Ok(_), Err(err)) => {
                     env.log(format!("backup ended and finish failed: {}", err));
                     env.log("removing unfinished backup");
-                    env.remove_backup()?;
+                    tools::runtime::block_in_place(|| env.remove_backup())?;
                     Err(err)
                 },
                 (Err(err), Err(_)) => {

src/api2/backup/environment.rs

@@ -1,6 +1,7 @@
 use anyhow::{bail, format_err, Error};
 use std::sync::{Arc, Mutex};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
+use nix::dir::Dir;
 
 use ::serde::{Serialize};
 use serde_json::{json, Value};
@@ -9,7 +10,7 @@ use proxmox::tools::digest_to_hex;
 use proxmox::tools::fs::{replace_file, CreateOptions};
 use proxmox::api::{RpcEnvironment, RpcEnvironmentType};
 
-use crate::api2::types::Userid;
+use crate::api2::types::Authid;
 use crate::backup::*;
 use crate::server::WorkerTask;
 use crate::server::formatter::*;
@@ -103,7 +104,7 @@ impl SharedBackupState {
 pub struct BackupEnvironment {
     env_type: RpcEnvironmentType,
     result_attributes: Value,
-    user: Userid,
+    auth_id: Authid,
     pub debug: bool,
     pub formatter: &'static OutputFormatter,
     pub worker: Arc<WorkerTask>,
@@ -116,7 +117,7 @@ pub struct BackupEnvironment {
 impl BackupEnvironment {
     pub fn new(
         env_type: RpcEnvironmentType,
-        user: Userid,
+        auth_id: Authid,
         worker: Arc<WorkerTask>,
         datastore: Arc<DataStore>,
         backup_dir: BackupDir,
@@ -136,7 +137,7 @@ impl BackupEnvironment {
         Self {
             result_attributes: json!({}),
             env_type,
-            user,
+            auth_id,
             worker,
             datastore,
             debug: false,
@@ -472,16 +473,11 @@ impl BackupEnvironment {
             bail!("backup does not contain valid files (file count == 0)");
         }
 
-        // check manifest
-        let mut manifest = self.datastore.load_manifest_json(&self.backup_dir)
-            .map_err(|err| format_err!("unable to load manifest blob - {}", err))?;
-
+        // check for valid manifest and store stats
         let stats = serde_json::to_value(state.backup_stat)?;
-
-        manifest["unprotected"]["chunk_upload_stats"] = stats;
-
-        self.datastore.store_manifest(&self.backup_dir, manifest)
-            .map_err(|err| format_err!("unable to store manifest blob - {}", err))?;
+        self.datastore.update_manifest(&self.backup_dir, |manifest| {
+            manifest.unprotected["chunk_upload_stats"] = stats;
+        }).map_err(|err| format_err!("unable to update manifest blob - {}", err))?;
 
         if let Some(base) = &self.last_backup {
             let path = self.datastore.snapshot_path(&base.backup_dir);
@@ -499,6 +495,55 @@ impl BackupEnvironment {
         Ok(())
     }
 
+    /// If verify-new is set on the datastore, this will run a new verify task
+    /// for the backup. If not, this will return and also drop the passed lock
+    /// immediately.
+    pub fn verify_after_complete(&self, snap_lock: Dir) -> Result<(), Error> {
+        self.ensure_finished()?;
+
+        if !self.datastore.verify_new() {
+            // no verify requested, do nothing
+            return Ok(());
+        }
+
+        let worker_id = format!("{}:{}/{}/{:08X}",
+            self.datastore.name(),
+            self.backup_dir.group().backup_type(),
+            self.backup_dir.group().backup_id(),
+            self.backup_dir.backup_time());
+
+        let datastore = self.datastore.clone();
+        let backup_dir = self.backup_dir.clone();
+
+        WorkerTask::new_thread(
+            "verify",
+            Some(worker_id),
+            self.auth_id.clone(),
+            false,
+            move |worker| {
+                worker.log("Automatically verifying newly added snapshot");
+
+                let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*16)));
+                let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
+
+                if !verify_backup_dir_with_lock(
+                    datastore,
+                    &backup_dir,
+                    verified_chunks,
+                    corrupt_chunks,
+                    worker.clone(),
+                    worker.upid().clone(),
+                    None,
+                    snap_lock,
+                )? {
+                    bail!("verification failed - please check the log for details");
+                }
+
+                Ok(())
+            },
+        ).map(|_| ())
+    }
+
     pub fn log<S: AsRef<str>>(&self, msg: S) {
         self.worker.log(msg);
     }
@@ -523,6 +568,12 @@ impl BackupEnvironment {
         Ok(())
     }
 
+    /// Return true if the finished flag is set
+    pub fn finished(&self) -> bool {
+        let state = self.state.lock().unwrap();
+        state.finished
+    }
+
     /// Remove complete backup
     pub fn remove_backup(&self) -> Result<(), Error> {
         let mut state = self.state.lock().unwrap();
@@ -548,12 +599,12 @@ impl RpcEnvironment for BackupEnvironment {
         self.env_type
     }
 
-    fn set_user(&mut self, _user: Option<String>) {
-        panic!("unable to change user");
+    fn set_auth_id(&mut self, _auth_id: Option<String>) {
+        panic!("unable to change auth_id");
     }
 
-    fn get_user(&self) -> Option<String> {
-        Some(self.user.to_string())
+    fn get_auth_id(&self) -> Option<String> {
+        Some(self.auth_id.to_string())
     }
 }
 

src/api2/config.rs

@@ -4,11 +4,13 @@ use proxmox::list_subdirs_api_method;
 pub mod datastore;
 pub mod remote;
 pub mod sync;
+pub mod verify;
 
 const SUBDIRS: SubdirMap = &[
     ("datastore", &datastore::ROUTER),
     ("remote", &remote::ROUTER),
     ("sync", &sync::ROUTER),
+    ("verify", &verify::ROUTER)
 ];
 
 pub const ROUTER: Router = Router::new()

src/api2/config/datastore.rs

@@ -5,6 +5,7 @@ use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, Router, RpcEnvironment, Permission};
+use proxmox::api::schema::parse_property_string;
 use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
@@ -12,6 +13,7 @@ use crate::backup::*;
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::datastore::{self, DataStoreConfig, DIR_NAME_SCHEMA};
 use crate::config::acl::{PRIV_DATASTORE_ALLOCATE, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY};
+use crate::server::jobstate;
 
 #[api(
     input: {
@@ -34,14 +36,14 @@ pub fn list_datastores(
 
     let (config, digest) = datastore::config()?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let user_info = CachedUserInfo::new()?;
 
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
     let list:Vec<DataStoreConfig> = config.convert_to_typed_array("datastore")?;
     let filter_by_privs = |store: &DataStoreConfig| {
-        let user_privs = user_info.lookup_privs(&userid, &["datastore", &store.name]);
+        let user_privs = user_info.lookup_privs(&auth_id, &["datastore", &store.name]);
         (user_privs & PRIV_DATASTORE_AUDIT) != 0
     };
 
@@ -67,6 +69,14 @@ pub fn list_datastores(
                 optional: true,
                 schema: SINGLE_LINE_COMMENT_SCHEMA,
             },
+            "notify-user": {
+                optional: true,
+                type: Userid,
+            },
+            "notify": {
+                optional: true,
+                schema: DATASTORE_NOTIFY_STRING_SCHEMA,
+            },
             "gc-schedule": {
                 optional: true,
                 schema: GC_SCHEDULE_SCHEMA,
@@ -75,10 +85,6 @@ pub fn list_datastores(
                 optional: true,
                 schema: PRUNE_SCHEDULE_SCHEMA,
             },
-            "verify-schedule": {
-                optional: true,
-                schema: VERIFY_SCHEDULE_SCHEMA,
-            },
             "keep-last": {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -131,9 +137,8 @@ pub fn create_datastore(param: Value) -> Result<(), Error> {
 
     datastore::save_config(&config)?;
 
-    crate::config::jobstate::create_state_file("prune", &datastore.name)?;
-    crate::config::jobstate::create_state_file("garbage_collection", &datastore.name)?;
-    crate::config::jobstate::create_state_file("verify", &datastore.name)?;
+    jobstate::create_state_file("prune", &datastore.name)?;
+    jobstate::create_state_file("garbage_collection", &datastore.name)?;
 
     Ok(())
 }
@@ -179,8 +184,6 @@ pub enum DeletableProperty {
     gc_schedule,
     /// Delete the prune job schedule.
     prune_schedule,
-    /// Delete the verify schedule property
-    verify_schedule,
     /// Delete the keep-last property
     keep_last,
     /// Delete the keep-hourly property
@@ -193,6 +196,12 @@ pub enum DeletableProperty {
     keep_monthly,
     /// Delete the keep-yearly property
     keep_yearly,
+    /// Delete the verify-new property
+    verify_new,
+    /// Delete the notify-user property
+    notify_user,
+    /// Delete the notify property
+    notify,
 }
 
 #[api(
@@ -206,6 +215,14 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: SINGLE_LINE_COMMENT_SCHEMA,
             },
+            "notify-user": {
+                optional: true,
+                type: Userid,
+            },
+            "notify": {
+                optional: true,
+                schema: DATASTORE_NOTIFY_STRING_SCHEMA,
+            },
             "gc-schedule": {
                 optional: true,
                 schema: GC_SCHEDULE_SCHEMA,
@@ -214,10 +231,6 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: PRUNE_SCHEDULE_SCHEMA,
             },
-            "verify-schedule": {
-                optional: true,
-                schema: VERIFY_SCHEDULE_SCHEMA,
-            },
             "keep-last": {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -242,6 +255,12 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_YEARLY,
             },
+            "verify-new": {
+                description: "If enabled, all new backups will be verified right after completion.",
+                type: bool,
+                optional: true,
+                default: false,
+            },
             delete: {
                 description: "List of properties to delete.",
                 type: Array,
@@ -266,13 +285,15 @@ pub fn update_datastore(
     comment: Option<String>,
     gc_schedule: Option<String>,
     prune_schedule: Option<String>,
-    verify_schedule: Option<String>,
     keep_last: Option<u64>,
     keep_hourly: Option<u64>,
     keep_daily: Option<u64>,
     keep_weekly: Option<u64>,
     keep_monthly: Option<u64>,
     keep_yearly: Option<u64>,
+    verify_new: Option<bool>,
+    notify: Option<String>,
+    notify_user: Option<Userid>,
     delete: Option<Vec<DeletableProperty>>,
     digest: Option<String>,
 ) -> Result<(), Error> {
@@ -295,13 +316,15 @@ pub fn update_datastore(
                 DeletableProperty::comment => { data.comment = None; },
                 DeletableProperty::gc_schedule => { data.gc_schedule = None; },
                 DeletableProperty::prune_schedule => { data.prune_schedule = None; },
-                DeletableProperty::verify_schedule => { data.verify_schedule = None; },
                 DeletableProperty::keep_last => { data.keep_last = None; },
                 DeletableProperty::keep_hourly => { data.keep_hourly = None; },
                 DeletableProperty::keep_daily => { data.keep_daily = None; },
                 DeletableProperty::keep_weekly => { data.keep_weekly = None; },
                 DeletableProperty::keep_monthly => { data.keep_monthly = None; },
                 DeletableProperty::keep_yearly => { data.keep_yearly = None; },
+                DeletableProperty::verify_new => { data.verify_new = None; },
+                DeletableProperty::notify => { data.notify = None; },
+                DeletableProperty::notify_user => { data.notify_user = None; },
             }
         }
     }
@@ -327,12 +350,6 @@ pub fn update_datastore(
         data.prune_schedule = prune_schedule;
     }
 
-    let mut verify_schedule_changed = false;
-    if verify_schedule.is_some() {
-        verify_schedule_changed = data.verify_schedule != verify_schedule;
-        data.verify_schedule = verify_schedule;
-    }
-
     if keep_last.is_some() { data.keep_last = keep_last; }
     if keep_hourly.is_some() { data.keep_hourly = keep_hourly; }
     if keep_daily.is_some() { data.keep_daily = keep_daily; }
@@ -340,6 +357,19 @@ pub fn update_datastore(
     if keep_monthly.is_some() { data.keep_monthly = keep_monthly; }
     if keep_yearly.is_some() { data.keep_yearly = keep_yearly; }
 
+    if let Some(notify_str) = notify {
+        let value = parse_property_string(&notify_str, &DatastoreNotify::API_SCHEMA)?;
+        let notify: DatastoreNotify = serde_json::from_value(value)?;
+        if let  DatastoreNotify { gc: None, verify: None, sync: None } = notify {
+            data.notify = None;
+        } else {
+            data.notify = Some(notify_str);
+        }
+    }
+    if verify_new.is_some() { data.verify_new = verify_new; }
+
+    if notify_user.is_some() { data.notify_user = notify_user; }
+
     config.set_data(&name, "datastore", &data)?;
 
     datastore::save_config(&config)?;
@@ -347,15 +377,11 @@ pub fn update_datastore(
     // we want to reset the statefiles, to avoid an immediate action in some cases
     // (e.g. going from monthly to weekly in the second week of the month)
     if gc_schedule_changed {
-        crate::config::jobstate::create_state_file("garbage_collection", &name)?;
+        jobstate::create_state_file("garbage_collection", &name)?;
     }
 
     if prune_schedule_changed {
-        crate::config::jobstate::create_state_file("prune", &name)?;
-    }
-
-    if verify_schedule_changed {
-        crate::config::jobstate::create_state_file("verify", &name)?;
+        jobstate::create_state_file("prune", &name)?;
     }
 
     Ok(())
@@ -398,9 +424,8 @@ pub fn delete_datastore(name: String, digest: Option<String>) -> Result<(), Erro
     datastore::save_config(&config)?;
 
     // ignore errors
-    let _ = crate::config::jobstate::remove_state_file("prune", &name);
-    let _ = crate::config::jobstate::remove_state_file("garbage_collection", &name);
-    let _ = crate::config::jobstate::remove_state_file("verify", &name);
+    let _ = jobstate::remove_state_file("prune", &name);
+    let _ = jobstate::remove_state_file("garbage_collection", &name);
 
     Ok(())
 }

src/api2/config/remote.rs

@@ -1,12 +1,14 @@
-use anyhow::{bail, Error};
+use anyhow::{bail, format_err, Error};
 use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
-use base64;
 
 use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
+use proxmox::http_err;
 use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
+use crate::client::{HttpClient, HttpClientOptions};
+use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::remote;
 use crate::config::acl::{PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY};
 
@@ -23,7 +25,8 @@ use crate::config::acl::{PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY};
         },
     },
     access: {
-        permission: &Permission::Privilege(&["remote"], PRIV_REMOTE_AUDIT, false),
+        description: "List configured remotes filtered by Remote.Audit privileges",
+        permission: &Permission::Anybody,
     },
 )]
 /// List all remotes
@@ -32,16 +35,25 @@ pub fn list_remotes(
     _info: &ApiMethod,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<remote::Remote>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let (config, digest) = remote::config()?;
 
     let mut list: Vec<remote::Remote> = config.convert_to_typed_array("remote")?;
-
     // don't return password in api
     for remote in &mut list {
         remote.password = "".to_string();
     }
 
+    let list = list
+        .into_iter()
+        .filter(|remote| {
+            let privs = user_info.lookup_privs(&auth_id, &["remote", &remote.name]);
+            privs & PRIV_REMOTE_AUDIT != 0
+        })
+        .collect();
+
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
     Ok(list)
 }
@@ -67,7 +79,7 @@ pub fn list_remotes(
                 default: 8007,
             },
             userid: {
-                type: Userid,
+                type: Authid,
             },
             password: {
                 schema: remote::REMOTE_PASSWORD_SCHEMA,
@@ -168,7 +180,7 @@ pub enum DeletableProperty {
             },
             userid: {
                 optional: true,
-                type: Userid,
+                type: Authid,
             },
             password: {
                 optional: true,
@@ -202,7 +214,7 @@ pub fn update_remote(
     comment: Option<String>,
     host: Option<String>,
     port: Option<u16>,
-    userid: Option<Userid>,
+    userid: Option<Authid>,
     password: Option<String>,
     fingerprint: Option<String>,
     delete: Option<Vec<DeletableProperty>>,
@@ -291,10 +303,83 @@ pub fn delete_remote(name: String, digest: Option<String>) -> Result<(), Error>
     Ok(())
 }
 
+/// Helper to get client for remote.cfg entry
+pub async fn remote_client(remote: remote::Remote) -> Result<HttpClient, Error> {
+    let options = HttpClientOptions::new()
+        .password(Some(remote.password.clone()))
+        .fingerprint(remote.fingerprint.clone());
+
+    let client = HttpClient::new(
+        &remote.host,
+        remote.port.unwrap_or(8007),
+        &remote.userid,
+        options)?;
+    let _auth_info = client.login() // make sure we can auth
+        .await
+        .map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?;
+
+    Ok(client)
+}
+
+
+#[api(
+    input: {
+        properties: {
+            name: {
+                schema: REMOTE_ID_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["remote", "{name}"], PRIV_REMOTE_AUDIT, false),
+    },
+    returns: {
+        description: "List the accessible datastores.",
+        type: Array,
+        items: {
+            description: "Datastore name and description.",
+            type: DataStoreListItem,
+        },
+    },
+)]
+/// List datastores of a remote.cfg entry
+pub async fn scan_remote_datastores(name: String) -> Result<Vec<DataStoreListItem>, Error> {
+    let (remote_config, _digest) = remote::config()?;
+    let remote: remote::Remote = remote_config.lookup("remote", &name)?;
+
+    let map_remote_err = |api_err| {
+        http_err!(INTERNAL_SERVER_ERROR,
+                  "failed to scan remote '{}' - {}",
+                  &name,
+                  api_err)
+    };
+
+    let client = remote_client(remote)
+        .await
+        .map_err(map_remote_err)?;
+    let api_res = client
+        .get("api2/json/admin/datastore", None)
+        .await
+        .map_err(map_remote_err)?;
+    let parse_res = match api_res.get("data") {
+        Some(data) => serde_json::from_value::<Vec<DataStoreListItem>>(data.to_owned()),
+        None => bail!("remote {} did not return any datastore list data", &name),
+    };
+
+    match parse_res {
+        Ok(parsed) => Ok(parsed),
+        Err(_) => bail!("Failed to parse remote scan api result."),
+    }
+}
+
+const SCAN_ROUTER: Router = Router::new()
+    .get(&API_METHOD_SCAN_REMOTE_DATASTORES);
+
 const ITEM_ROUTER: Router = Router::new()
     .get(&API_METHOD_READ_REMOTE)
     .put(&API_METHOD_UPDATE_REMOTE)
-    .delete(&API_METHOD_DELETE_REMOTE);
+    .delete(&API_METHOD_DELETE_REMOTE)
+    .subdirs(&[("scan", &SCAN_ROUTER)]);
 
 pub const ROUTER: Router = Router::new()
     .get(&API_METHOD_LIST_REMOTES)

src/api2/config/sync.rs

@@ -2,13 +2,73 @@ use anyhow::{bail, Error};
 use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
 
-use proxmox::api::{api, Router, RpcEnvironment};
+use proxmox::api::{api, Permission, Router, RpcEnvironment};
 use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
+
+use crate::config::acl::{
+    PRIV_DATASTORE_AUDIT,
+    PRIV_DATASTORE_BACKUP,
+    PRIV_DATASTORE_MODIFY,
+    PRIV_DATASTORE_PRUNE,
+    PRIV_REMOTE_AUDIT,
+    PRIV_REMOTE_READ,
+};
+
+use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::sync::{self, SyncJobConfig};
 
-// fixme: add access permissions
+pub fn check_sync_job_read_access(
+    user_info: &CachedUserInfo,
+    auth_id: &Authid,
+    job: &SyncJobConfig,
+) -> bool {
+    let datastore_privs = user_info.lookup_privs(&auth_id, &["datastore", &job.store]);
+    if datastore_privs & PRIV_DATASTORE_AUDIT == 0 {
+        return false;
+    }
+
+    let remote_privs = user_info.lookup_privs(&auth_id, &["remote", &job.remote]);
+    remote_privs & PRIV_REMOTE_AUDIT != 0
+}
+
+// user can run the corresponding pull job
+pub fn check_sync_job_modify_access(
+    user_info: &CachedUserInfo,
+    auth_id: &Authid,
+    job: &SyncJobConfig,
+) -> bool {
+    let datastore_privs = user_info.lookup_privs(&auth_id, &["datastore", &job.store]);
+    if datastore_privs & PRIV_DATASTORE_BACKUP == 0 {
+        return false;
+    }
+
+    if let Some(true) = job.remove_vanished {
+        if datastore_privs & PRIV_DATASTORE_PRUNE == 0 {
+            return false;
+        }
+    }
+
+    let correct_owner = match job.owner {
+        Some(ref owner) => {
+            owner == auth_id
+                || (owner.is_token()
+                    && !auth_id.is_token()
+                    && owner.user() == auth_id.user())
+        },
+        // default sync owner
+        None => auth_id == Authid::root_auth_id(),
+    };
+
+    // same permission as changing ownership after syncing
+    if !correct_owner && datastore_privs & PRIV_DATASTORE_MODIFY == 0 {
+        return false;
+    }
+
+    let remote_privs = user_info.lookup_privs(&auth_id, &["remote", &job.remote, &job.remote_store]);
+    remote_privs & PRIV_REMOTE_READ != 0
+}
 
 #[api(
     input: {
@@ -19,12 +79,18 @@ use crate::config::sync::{self, SyncJobConfig};
         type: Array,
         items: { type: sync::SyncJobConfig },
     },
+    access: {
+        description: "Limited to sync job entries where user has Datastore.Audit on target datastore, and Remote.Audit on source remote.",
+        permission: &Permission::Anybody,
+    },
 )]
 /// List all sync jobs
 pub fn list_sync_jobs(
     _param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<SyncJobConfig>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let (config, digest) = sync::config()?;
 
@@ -32,7 +98,11 @@ pub fn list_sync_jobs(
 
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
-    Ok(list)
+    let list = list
+        .into_iter()
+        .filter(|sync_job| check_sync_job_read_access(&user_info, &auth_id, &sync_job))
+        .collect();
+   Ok(list)
 }
 
 #[api(
@@ -45,6 +115,10 @@ pub fn list_sync_jobs(
             store: {
                 schema: DATASTORE_SCHEMA,
             },
+            owner: {
+                type: Authid,
+                optional: true,
+            },
             remote: {
                 schema: REMOTE_ID_SCHEMA,
             },
@@ -65,13 +139,25 @@ pub fn list_sync_jobs(
             },
         },
     },
+    access: {
+        description: "User needs Datastore.Backup on target datastore, and Remote.Read on source remote. Additionally, remove_vanished requires Datastore.Prune, and any owner other than the user themselves requires Datastore.Modify",
+        permission: &Permission::Anybody,
+    },
 )]
 /// Create a new sync job.
-pub fn create_sync_job(param: Value) -> Result<(), Error> {
+pub fn create_sync_job(
+    param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
     let sync_job: sync::SyncJobConfig = serde_json::from_value(param.clone())?;
+    if !check_sync_job_modify_access(&user_info, &auth_id, &sync_job) {
+        bail!("permission check failed");
+    }
 
     let (mut config, _digest) = sync::config()?;
 
@@ -83,7 +169,7 @@ pub fn create_sync_job(param: Value) -> Result<(), Error> {
 
     sync::save_config(&config)?;
 
-    crate::config::jobstate::create_state_file("syncjob", &sync_job.id)?;
+    crate::server::jobstate::create_state_file("syncjob", &sync_job.id)?;
 
     Ok(())
 }
@@ -100,15 +186,26 @@ pub fn create_sync_job(param: Value) -> Result<(), Error> {
         description: "The sync job configuration.",
         type: sync::SyncJobConfig,
     },
+    access: {
+        description: "Limited to sync job entries where user has Datastore.Audit on target datastore, and Remote.Audit on source remote.",
+        permission: &Permission::Anybody,
+    },
 )]
 /// Read a sync job configuration.
 pub fn read_sync_job(
     id: String,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<SyncJobConfig, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
     let (config, digest) = sync::config()?;
 
     let sync_job = config.lookup("sync", &id)?;
+    if !check_sync_job_read_access(&user_info, &auth_id, &sync_job) {
+        bail!("permission check failed");
+    }
+
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
 
     Ok(sync_job)
@@ -120,6 +217,8 @@ pub fn read_sync_job(
 #[allow(non_camel_case_types)]
 /// Deletable property name
 pub enum DeletableProperty {
+    /// Delete the owner property.
+    owner,
     /// Delete the comment property.
     comment,
     /// Delete the job schedule.
@@ -139,6 +238,10 @@ pub enum DeletableProperty {
                 schema: DATASTORE_SCHEMA,
                 optional: true,
             },
+            owner: {
+                type: Authid,
+                optional: true,
+            },
             remote: {
                 schema: REMOTE_ID_SCHEMA,
                 optional: true,
@@ -173,11 +276,16 @@ pub enum DeletableProperty {
             },
         },
     },
+    access: {
+        permission: &Permission::Anybody,
+        description: "User needs Datastore.Backup on target datastore, and Remote.Read on source remote. Additionally, remove_vanished requires Datastore.Prune, and any owner other than the user themselves requires Datastore.Modify",
+    },
 )]
 /// Update sync job config.
 pub fn update_sync_job(
     id: String,
     store: Option<String>,
+    owner: Option<Authid>,
     remote: Option<String>,
     remote_store: Option<String>,
     remove_vanished: Option<bool>,
@@ -185,7 +293,10 @@ pub fn update_sync_job(
     schedule: Option<String>,
     delete: Option<Vec<DeletableProperty>>,
     digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
@@ -202,6 +313,7 @@ pub fn update_sync_job(
      if let Some(delete) = delete {
         for delete_prop in delete {
             match delete_prop {
+                DeletableProperty::owner => { data.owner = None; },
                 DeletableProperty::comment => { data.comment = None; },
                 DeletableProperty::schedule => { data.schedule = None; },
                 DeletableProperty::remove_vanished => { data.remove_vanished = None; },
@@ -221,11 +333,15 @@ pub fn update_sync_job(
     if let Some(store) = store { data.store = store; }
     if let Some(remote) = remote { data.remote = remote; }
     if let Some(remote_store) = remote_store { data.remote_store = remote_store; }
-        
-    
+    if let Some(owner) = owner { data.owner = Some(owner); }
+
     if schedule.is_some() { data.schedule = schedule; }
     if remove_vanished.is_some() { data.remove_vanished = remove_vanished; }
 
+    if !check_sync_job_modify_access(&user_info, &auth_id, &data) {
+        bail!("permission check failed");
+    }
+
     config.set_data(&id, "sync", &data)?;
 
     sync::save_config(&config)?;
@@ -246,9 +362,19 @@ pub fn update_sync_job(
             },
         },
     },
+    access: {
+        permission: &Permission::Anybody,
+        description: "User needs Datastore.Backup on target datastore, and Remote.Read on source remote. Additionally, remove_vanished requires Datastore.Prune, and any owner other than the user themselves requires Datastore.Modify",
+    },
 )]
 /// Remove a sync job configuration
-pub fn delete_sync_job(id: String, digest: Option<String>) -> Result<(), Error> {
+pub fn delete_sync_job(
+    id: String,
+    digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
 
     let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
 
@@ -259,14 +385,19 @@ pub fn delete_sync_job(id: String, digest: Option<String>) -> Result<(), Error>
         crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
     }
 
-    match config.sections.get(&id) {
-        Some(_) => { config.sections.remove(&id); },
-        None => bail!("job '{}' does not exist.", id),
-    }
+    match config.lookup("sync", &id) {
+        Ok(job) => {
+            if !check_sync_job_modify_access(&user_info, &auth_id, &job) {
+                bail!("permission check failed");
+            }
+            config.sections.remove(&id);
+        },
+        Err(_) => { bail!("job '{}' does not exist.", id) },
+    };
 
     sync::save_config(&config)?;
 
-    crate::config::jobstate::remove_state_file("syncjob", &id)?;
+    crate::server::jobstate::remove_state_file("syncjob", &id)?;
 
     Ok(())
 }
@@ -280,3 +411,116 @@ pub const ROUTER: Router = Router::new()
     .get(&API_METHOD_LIST_SYNC_JOBS)
     .post(&API_METHOD_CREATE_SYNC_JOB)
     .match_all("id", &ITEM_ROUTER);
+
+
+#[test]
+fn sync_job_access_test() -> Result<(), Error> {
+    let (user_cfg, _) = crate::config::user::test_cfg_from_str(r###"
+user: noperm@pbs
+
+user: read@pbs
+
+user: write@pbs
+
+"###).expect("test user.cfg is not parsable");
+    let acl_tree = crate::config::acl::AclTree::from_raw(r###"
+acl:1:/datastore/localstore1:read@pbs,write@pbs:DatastoreAudit
+acl:1:/datastore/localstore1:write@pbs:DatastoreBackup
+acl:1:/datastore/localstore2:write@pbs:DatastorePowerUser
+acl:1:/datastore/localstore3:write@pbs:DatastoreAdmin
+acl:1:/remote/remote1:read@pbs,write@pbs:RemoteAudit
+acl:1:/remote/remote1/remotestore1:write@pbs:RemoteSyncOperator
+"###).expect("test acl.cfg is not parsable");
+
+    let user_info = CachedUserInfo::test_new(user_cfg, acl_tree);
+
+    let root_auth_id = Authid::root_auth_id();
+
+    let no_perm_auth_id: Authid = "noperm@pbs".parse()?;
+    let read_auth_id: Authid = "read@pbs".parse()?;
+    let write_auth_id: Authid = "write@pbs".parse()?;
+
+    let mut job = SyncJobConfig {
+        id: "regular".to_string(),
+        remote: "remote0".to_string(),
+        remote_store: "remotestore1".to_string(),
+        store: "localstore0".to_string(),
+        owner: Some(write_auth_id.clone()),
+        comment: None,
+        remove_vanished: None,
+        schedule: None,
+    };
+
+    // should work without ACLs
+    assert_eq!(check_sync_job_read_access(&user_info, &root_auth_id, &job), true);
+    assert_eq!(check_sync_job_modify_access(&user_info, &root_auth_id, &job), true);
+
+    // user without permissions must fail
+    assert_eq!(check_sync_job_read_access(&user_info, &no_perm_auth_id, &job), false);
+    assert_eq!(check_sync_job_modify_access(&user_info, &no_perm_auth_id, &job), false);
+
+    // reading without proper read permissions on either remote or local must fail
+    assert_eq!(check_sync_job_read_access(&user_info, &read_auth_id, &job), false);
+
+    // reading without proper read permissions on local end must fail
+    job.remote = "remote1".to_string();
+    assert_eq!(check_sync_job_read_access(&user_info, &read_auth_id, &job), false);
+
+    // reading without proper read permissions on remote end must fail
+    job.remote = "remote0".to_string();
+    job.store = "localstore1".to_string();
+    assert_eq!(check_sync_job_read_access(&user_info, &read_auth_id, &job), false);
+
+    // writing without proper write permissions on either end must fail
+    job.store = "localstore0".to_string();
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
+
+    // writing without proper write permissions on local end must fail
+    job.remote = "remote1".to_string();
+
+    // writing without proper write permissions on remote end must fail
+    job.remote = "remote0".to_string();
+    job.store = "localstore1".to_string();
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
+
+    // reset remote to one where users have access
+    job.remote = "remote1".to_string();
+
+    // user with read permission can only read, but not modify/run
+    assert_eq!(check_sync_job_read_access(&user_info, &read_auth_id, &job), true);
+    job.owner = Some(read_auth_id.clone());
+    assert_eq!(check_sync_job_modify_access(&user_info, &read_auth_id, &job), false);
+    job.owner = None;
+    assert_eq!(check_sync_job_modify_access(&user_info, &read_auth_id, &job), false);
+    job.owner = Some(write_auth_id.clone());
+    assert_eq!(check_sync_job_modify_access(&user_info, &read_auth_id, &job), false);
+
+    // user with simple write permission can modify/run
+    assert_eq!(check_sync_job_read_access(&user_info, &write_auth_id, &job), true);
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), true);
+
+    // but can't modify/run with deletion
+    job.remove_vanished = Some(true);
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
+
+    // unless they have Datastore.Prune as well
+    job.store = "localstore2".to_string();
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), true);
+
+    // changing owner is not possible
+    job.owner = Some(read_auth_id.clone());
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
+
+    // also not to the default 'root@pam'
+    job.owner = None;
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), false);
+
+    // unless they have Datastore.Modify as well
+    job.store = "localstore3".to_string();
+    job.owner = Some(read_auth_id.clone());
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), true);
+    job.owner = None;
+    assert_eq!(check_sync_job_modify_access(&user_info, &write_auth_id, &job), true);
+
+    Ok(())
+}

src/api2/config/verify.rs

@@ -0,0 +1,347 @@
+use anyhow::{bail, Error};
+use serde_json::Value;
+use ::serde::{Deserialize, Serialize};
+
+use proxmox::api::{api, Permission, Router, RpcEnvironment};
+use proxmox::tools::fs::open_file_locked;
+
+use crate::api2::types::*;
+
+use crate::config::acl::{
+    PRIV_DATASTORE_AUDIT,
+    PRIV_DATASTORE_VERIFY,
+};
+
+use crate::config::cached_user_info::CachedUserInfo;
+
+use crate::config::verify::{self, VerificationJobConfig};
+
+#[api(
+    input: {
+        properties: {},
+    },
+    returns: {
+        description: "List configured jobs.",
+        type: Array,
+        items: { type: verify::VerificationJobConfig },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Audit or Datastore.Verify on datastore.",
+    },
+)]
+/// List all verification jobs
+pub fn list_verification_jobs(
+    _param: Value,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<VerificationJobConfig>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_VERIFY;
+
+    let (config, digest) = verify::config()?;
+
+    let list = config.convert_to_typed_array("verification")?;
+
+    let list = list.into_iter()
+        .filter(|job: &VerificationJobConfig| {
+            let privs = user_info.lookup_privs(&auth_id, &["datastore", &job.store]);
+
+            privs & required_privs != 00
+        }).collect();
+
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+    Ok(list)
+}
+
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            store: {
+                schema: DATASTORE_SCHEMA,
+            },
+            "ignore-verified": {
+                optional: true,
+                schema: IGNORE_VERIFIED_BACKUPS_SCHEMA,
+            },
+            "outdated-after": {
+                optional: true,
+                schema: VERIFICATION_OUTDATED_AFTER_SCHEMA,
+            },
+            comment: {
+                optional: true,
+                schema: SINGLE_LINE_COMMENT_SCHEMA,
+            },
+            schedule: {
+                optional: true,
+                schema: VERIFICATION_SCHEDULE_SCHEMA,
+            },
+        }
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Verify on job's datastore.",
+    },
+)]
+/// Create a new verification job.
+pub fn create_verification_job(
+    param: Value,
+    rpcenv: &mut dyn RpcEnvironment
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let verification_job: verify::VerificationJobConfig = serde_json::from_value(param.clone())?;
+
+    user_info.check_privs(&auth_id, &["datastore", &verification_job.store], PRIV_DATASTORE_VERIFY, false)?;
+
+    let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
+    let (mut config, _digest) = verify::config()?;
+
+    if let Some(_) = config.sections.get(&verification_job.id) {
+        bail!("job '{}' already exists.", verification_job.id);
+    }
+
+    config.set_data(&verification_job.id, "verification", &verification_job)?;
+
+    verify::save_config(&config)?;
+
+    crate::server::jobstate::create_state_file("verificationjob", &verification_job.id)?;
+
+    Ok(())
+}
+
+#[api(
+   input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+        },
+    },
+    returns: {
+        description: "The verification job configuration.",
+        type: verify::VerificationJobConfig,
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Audit or Datastore.Verify on job's datastore.",
+    },
+)]
+/// Read a verification job configuration.
+pub fn read_verification_job(
+    id: String,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<VerificationJobConfig, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let (config, digest) = verify::config()?;
+
+    let verification_job: verify::VerificationJobConfig = config.lookup("verification", &id)?;
+
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_VERIFY;
+    user_info.check_privs(&auth_id, &["datastore", &verification_job.store], required_privs, true)?;
+
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+    Ok(verification_job)
+}
+
+#[api()]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all="kebab-case")]
+/// Deletable property name
+pub enum DeletableProperty {
+    /// Delete the ignore verified property.
+    IgnoreVerified,
+    /// Delete the comment property.
+    Comment,
+    /// Delete the job schedule.
+    Schedule,
+    /// Delete outdated after property.
+    OutdatedAfter
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            store: {
+                optional: true,
+                schema: DATASTORE_SCHEMA,
+            },
+            "ignore-verified": {
+                optional: true,
+                schema: IGNORE_VERIFIED_BACKUPS_SCHEMA,
+            },
+            "outdated-after": {
+                optional: true,
+                schema: VERIFICATION_OUTDATED_AFTER_SCHEMA,
+            },
+            comment: {
+                optional: true,
+                schema: SINGLE_LINE_COMMENT_SCHEMA,
+            },
+            schedule: {
+                optional: true,
+                schema: VERIFICATION_SCHEDULE_SCHEMA,
+            },
+            delete: {
+                description: "List of properties to delete.",
+                type: Array,
+                optional: true,
+                items: {
+                    type: DeletableProperty,
+                }
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Verify on job's datastore.",
+    },
+)]
+/// Update verification job config.
+pub fn update_verification_job(
+    id: String,
+    store: Option<String>,
+    ignore_verified: Option<bool>,
+    outdated_after: Option<i64>,
+    comment: Option<String>,
+    schedule: Option<String>,
+    delete: Option<Vec<DeletableProperty>>,
+    digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
+    // pass/compare digest
+    let (mut config, expected_digest) = verify::config()?;
+
+    if let Some(ref digest) = digest {
+        let digest = proxmox::tools::hex_to_digest(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    let mut data: verify::VerificationJobConfig = config.lookup("verification", &id)?;
+
+    // check existing store
+    user_info.check_privs(&auth_id, &["datastore", &data.store], PRIV_DATASTORE_VERIFY, true)?;
+
+    if let Some(delete) = delete {
+        for delete_prop in delete {
+            match delete_prop {
+                DeletableProperty::IgnoreVerified => { data.ignore_verified = None; },
+                DeletableProperty::OutdatedAfter => { data.outdated_after = None; },
+                DeletableProperty::Comment => { data.comment = None; },
+                DeletableProperty::Schedule => { data.schedule = None; },
+            }
+        }
+    }
+
+    if let Some(comment) = comment {
+        let comment = comment.trim().to_string();
+        if comment.is_empty() {
+            data.comment = None;
+        } else {
+            data.comment = Some(comment);
+        }
+    }
+
+    if let Some(store) = store {
+        // check new store
+        user_info.check_privs(&auth_id, &["datastore", &store], PRIV_DATASTORE_VERIFY, true)?;
+        data.store = store;
+    }
+
+
+    if ignore_verified.is_some() { data.ignore_verified = ignore_verified; }
+    if outdated_after.is_some() { data.outdated_after = outdated_after; }
+    if schedule.is_some() { data.schedule = schedule; }
+
+    config.set_data(&id, "verification", &data)?;
+
+    verify::save_config(&config)?;
+
+    Ok(())
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Verify on job's datastore.",
+    },
+)]
+/// Remove a verification job configuration
+pub fn delete_verification_job(
+    id: String,
+    digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
+
+    let (mut config, expected_digest) = verify::config()?;
+
+    let job: verify::VerificationJobConfig = config.lookup("verification", &id)?;
+    user_info.check_privs(&auth_id, &["datastore", &job.store], PRIV_DATASTORE_VERIFY, true)?;
+
+    if let Some(ref digest) = digest {
+        let digest = proxmox::tools::hex_to_digest(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    match config.sections.get(&id) {
+        Some(_) => { config.sections.remove(&id); },
+        None => bail!("job '{}' does not exist.", id),
+    }
+
+    verify::save_config(&config)?;
+
+    crate::server::jobstate::remove_state_file("verificationjob", &id)?;
+
+    Ok(())
+}
+
+const ITEM_ROUTER: Router = Router::new()
+    .get(&API_METHOD_READ_VERIFICATION_JOB)
+    .put(&API_METHOD_UPDATE_VERIFICATION_JOB)
+    .delete(&API_METHOD_DELETE_VERIFICATION_JOB);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_VERIFICATION_JOBS)
+    .post(&API_METHOD_CREATE_VERIFICATION_JOB)
+    .match_all("id", &ITEM_ROUTER);

src/api2/node.rs

@@ -24,20 +24,21 @@ use crate::server::WorkerTask;
 use crate::tools;
 use crate::tools::ticket::{self, Empty, Ticket};
 
+pub mod apt;
 pub mod disks;
 pub mod dns;
 pub mod network;
 pub mod tasks;
+pub mod subscription;
 
 pub(crate) mod rrd;
 
-mod apt;
 mod journal;
 mod services;
 mod status;
-mod subscription;
 mod syslog;
 mod time;
+mod report;
 
 pub const SHELL_CMD_SCHEMA: Schema = StringSchema::new("The command to run.")
     .format(&ApiStringFormat::Enum(&[
@@ -91,10 +92,12 @@ async fn termproxy(
     cmd: Option<String>,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
+    // intentionally user only for now
     let userid: Userid = rpcenv
-        .get_user()
+        .get_auth_id()
         .ok_or_else(|| format_err!("unknown user"))?
         .parse()?;
+    let auth_id = Authid::from(userid.clone());
 
     if userid.realm() != "pam" {
         bail!("only pam users can use the console");
@@ -137,7 +140,7 @@ async fn termproxy(
     let upid = WorkerTask::spawn(
         "termproxy",
         None,
-        userid,
+        auth_id,
         false,
         move |worker| async move {
             // move inside the worker so that it survives and does not close the port
@@ -272,7 +275,8 @@ fn upgrade_to_websocket(
     rpcenv: Box<dyn RpcEnvironment>,
 ) -> ApiResponseFuture {
     async move {
-        let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+        // intentionally user only for now
+        let userid: Userid = rpcenv.get_auth_id().unwrap().parse()?;
         let ticket = tools::required_string_param(&param, "vncticket")?;
         let port: u16 = tools::required_integer_param(&param, "port")? as u16;
 
@@ -307,6 +311,7 @@ pub const SUBDIRS: SubdirMap = &[
     ("dns", &dns::ROUTER),
     ("journal", &journal::ROUTER),
     ("network", &network::ROUTER),
+    ("report", &report::ROUTER),
     ("rrd", &rrd::ROUTER),
     ("services", &services::ROUTER),
     ("status", &status::ROUTER),

src/api2/node/apt.rs

@@ -1,186 +1,15 @@
-use apt_pkg_native::Cache;
-use anyhow::{Error, bail};
+use anyhow::{Error, bail, format_err};
 use serde_json::{json, Value};
 
-use proxmox::{list_subdirs_api_method, const_regex};
+use proxmox::list_subdirs_api_method;
 use proxmox::api::{api, RpcEnvironment, RpcEnvironmentType, Permission};
 use proxmox::api::router::{Router, SubdirMap};
 
 use crate::server::WorkerTask;
+use crate::tools::{apt, http};
 
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
-use crate::api2::types::{APTUpdateInfo, NODE_SCHEMA, Userid, UPID_SCHEMA};
-
-const_regex! {
-    VERSION_EPOCH_REGEX = r"^\d+:";
-    FILENAME_EXTRACT_REGEX = r"^.*/.*?_(.*)_Packages$";
-}
-
-// FIXME: Replace with call to 'apt changelog <pkg> --print-uris'. Currently
-// not possible as our packages do not have a URI set in their Release file
-fn get_changelog_url(
-    package: &str,
-    filename: &str,
-    source_pkg: &str,
-    version: &str,
-    source_version: &str,
-    origin: &str,
-    component: &str,
-) -> Result<String, Error> {
-    if origin == "" {
-        bail!("no origin available for package {}", package);
-    }
-
-    if origin == "Debian" {
-        let source_version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(source_version, "");
-
-        let prefix = if source_pkg.starts_with("lib") {
-            source_pkg.get(0..4)
-        } else {
-            source_pkg.get(0..1)
-        };
-
-        let prefix = match prefix {
-            Some(p) => p,
-            None => bail!("cannot get starting characters of package name '{}'", package)
-        };
-
-        // note: security updates seem to not always upload a changelog for
-        // their package version, so this only works *most* of the time
-        return Ok(format!("https://metadata.ftp-master.debian.org/changelogs/main/{}/{}/{}_{}_changelog",
-                          prefix, source_pkg, source_pkg, source_version));
-
-    } else if origin == "Proxmox" {
-        let version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(version, "");
-
-        let base = match (FILENAME_EXTRACT_REGEX.regex_obj)().captures(filename) {
-            Some(captures) => {
-                let base_capture = captures.get(1);
-                match base_capture {
-                    Some(base_underscore) => base_underscore.as_str().replace("_", "/"),
-                    None => bail!("incompatible filename, cannot find regex group")
-                }
-            },
-            None => bail!("incompatible filename, doesn't match regex")
-        };
-
-        return Ok(format!("http://download.proxmox.com/{}/{}_{}.changelog",
-                          base, package, version));
-    }
-
-    bail!("unknown origin ({}) or component ({})", origin, component)
-}
-
-fn list_installed_apt_packages<F: Fn(&str, &str, &str) -> bool>(filter: F)
-    -> Vec<APTUpdateInfo> {
-
-    let mut ret = Vec::new();
-
-    // note: this is not an 'apt update', it just re-reads the cache from disk
-    let mut cache = Cache::get_singleton();
-    cache.reload();
-
-    let mut cache_iter = cache.iter();
-
-    loop {
-        let view = match cache_iter.next() {
-            Some(view) => view,
-            None => break
-        };
-
-        let current_version = match view.current_version() {
-            Some(vers) => vers,
-            None => continue
-        };
-        let candidate_version = match view.candidate_version() {
-            Some(vers) => vers,
-            // if there's no candidate (i.e. no update) get info of currently
-            // installed version instead
-            None => current_version.clone()
-        };
-
-        let package = view.name();
-        if filter(&package, &current_version, &candidate_version) {
-            let mut origin_res = "unknown".to_owned();
-            let mut section_res = "unknown".to_owned();
-            let mut priority_res = "unknown".to_owned();
-            let mut change_log_url = "".to_owned();
-            let mut short_desc = package.clone();
-            let mut long_desc = "".to_owned();
-
-            // get additional information via nested APT 'iterators'
-            let mut view_iter = view.versions();
-            while let Some(ver) = view_iter.next() {
-                if ver.version() == candidate_version {
-                    if let Some(section) = ver.section() {
-                        section_res = section;
-                    }
-
-                    if let Some(prio) = ver.priority_type() {
-                        priority_res = prio;
-                    }
-
-                    // assume every package has only one origin file (not
-                    // origin, but origin *file*, for some reason those seem to
-                    // be different concepts in APT)
-                    let mut origin_iter = ver.origin_iter();
-                    let origin = origin_iter.next();
-                    if let Some(origin) = origin {
-
-                        if let Some(sd) = origin.short_desc() {
-                            short_desc = sd;
-                        }
-
-                        if let Some(ld) = origin.long_desc() {
-                            long_desc = ld;
-                        }
-
-                        // the package files appear in priority order, meaning
-                        // the one for the candidate version is first
-                        let mut pkg_iter = origin.file();
-                        let pkg_file = pkg_iter.next();
-                        if let Some(pkg_file) = pkg_file {
-                            if let Some(origin_name) = pkg_file.origin() {
-                                origin_res = origin_name;
-                            }
-
-                            let filename = pkg_file.file_name();
-                            let source_pkg = ver.source_package();
-                            let source_ver = ver.source_version();
-                            let component = pkg_file.component();
-
-                            // build changelog URL from gathered information
-                            // ignore errors, use empty changelog instead
-                            let url = get_changelog_url(&package, &filename, &source_pkg,
-                                &candidate_version, &source_ver, &origin_res, &component);
-                            if let Ok(url) = url {
-                                change_log_url = url;
-                            }
-                        }
-                    }
-
-                    break;
-                }
-            }
-
-            let info = APTUpdateInfo {
-                package,
-                title: short_desc,
-                arch: view.arch(),
-                description: long_desc,
-                change_log_url,
-                origin: origin_res,
-                version: candidate_version,
-                old_version: current_version,
-                priority: priority_res,
-                section: section_res,
-            };
-            ret.push(info);
-        }
-    }
-
-    return ret;
-}
+use crate::api2::types::{Authid, APTUpdateInfo, NODE_SCHEMA, UPID_SCHEMA};
 
 #[api(
     input: {
@@ -193,16 +22,60 @@ fn list_installed_apt_packages<F: Fn(&str, &str, &str) -> bool>(filter: F)
     returns: {
         description: "A list of packages with available updates.",
         type: Array,
-        items: { type: APTUpdateInfo },
+        items: {
+            type: APTUpdateInfo
+        },
     },
+    protected: true,
     access: {
         permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
     },
 )]
 /// List available APT updates
 fn apt_update_available(_param: Value) -> Result<Value, Error> {
-    let ret = list_installed_apt_packages(|_pkg, cur_ver, can_ver| cur_ver != can_ver);
-    Ok(json!(ret))
+
+    match apt::pkg_cache_expired() {
+        Ok(false) => {
+            if let Ok(Some(cache)) = apt::read_pkg_state() {
+                return Ok(json!(cache.package_status));
+            }
+        },
+        _ => (),
+    }
+
+    let cache = apt::update_cache()?;
+
+    return Ok(json!(cache.package_status));
+}
+
+fn do_apt_update(worker: &WorkerTask, quiet: bool) -> Result<(), Error> {
+    if !quiet { worker.log("starting apt-get update") }
+
+    // TODO: set proxy /etc/apt/apt.conf.d/76pbsproxy like PVE
+
+    let mut command = std::process::Command::new("apt-get");
+    command.arg("update");
+
+    // apt "errors" quite easily, and run_command is a bit rigid, so handle this inline for now.
+    let output = command.output()
+        .map_err(|err| format_err!("failed to execute {:?} - {}", command, err))?;
+
+    if !quiet {
+        worker.log(String::from_utf8(output.stdout)?);
+    }
+
+    // TODO: improve run_command to allow outputting both, stderr and stdout
+    if !output.status.success() {
+        if output.status.code().is_some() {
+            let msg = String::from_utf8(output.stderr)
+                .map(|m| if m.is_empty() { String::from("no error message") } else { m })
+                .unwrap_or_else(|_| String::from("non utf8 error message (suppressed)"));
+            worker.warn(msg);
+        } else {
+            bail!("terminated by signal");
+        }
+    }
+    Ok(())
 }
 
 #[api(
@@ -212,6 +85,13 @@ fn apt_update_available(_param: Value) -> Result<Value, Error> {
             node: {
                 schema: NODE_SCHEMA,
             },
+            notify: {
+                type: bool,
+                description: r#"Send notification mail about new package updates availanle to the
+                    email address configured for 'root@pam')."#,
+                optional: true,
+                default: false,
+            },
             quiet: {
                 description: "Only produces output suitable for logging, omitting progress indicators.",
                 type: bool,
@@ -229,26 +109,46 @@ fn apt_update_available(_param: Value) -> Result<Value, Error> {
 )]
 /// Update the APT database
 pub fn apt_update_database(
+    notify: Option<bool>,
     quiet: Option<bool>,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<String, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
+    // FIXME: change to non-option in signature and drop below once we have proxmox-api-macro 0.2.3
     let quiet = quiet.unwrap_or(API_METHOD_APT_UPDATE_DATABASE_PARAM_DEFAULT_QUIET);
+    let notify = notify.unwrap_or(API_METHOD_APT_UPDATE_DATABASE_PARAM_DEFAULT_NOTIFY);
 
-    let upid_str = WorkerTask::new_thread("aptupdate", None, userid, to_stdout, move |worker| {
-        if !quiet { worker.log("starting apt-get update") }
+    let upid_str = WorkerTask::new_thread("aptupdate", None, auth_id, to_stdout, move |worker| {
+        do_apt_update(&worker, quiet)?;
 
-        // TODO: set proxy /etc/apt/apt.conf.d/76pbsproxy like PVE
+        let mut cache = apt::update_cache()?;
 
-        let mut command = std::process::Command::new("apt-get");
-        command.arg("update");
+        if notify {
+            let mut notified = match cache.notified {
+                Some(notified) => notified,
+                None => std::collections::HashMap::new(),
+            };
+            let mut to_notify: Vec<&APTUpdateInfo> = Vec::new();
 
-        let output = crate::tools::run_command(command, None)?;
-        if !quiet { worker.log(output) }
-
-        // TODO: add mail notify for new updates like PVE
+            for pkg in &cache.package_status {
+                match notified.insert(pkg.package.to_owned(), pkg.version.to_owned()) {
+                    Some(notified_version) => {
+                        if notified_version != pkg.version {
+                            to_notify.push(pkg);
+                        }
+                    },
+                    None => to_notify.push(pkg),
+                }
+            }
+            if !to_notify.is_empty() {
+                to_notify.sort_unstable_by_key(|k| &k.package);
+                crate::server::send_updates_available(&to_notify)?;
+            }
+            cache.notified = Some(notified);
+            apt::write_pkg_cache(&cache)?;
+        }
 
         Ok(())
     })?;
@@ -256,7 +156,67 @@ pub fn apt_update_database(
     Ok(upid_str)
 }
 
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            name: {
+                description: "Package name to get changelog of.",
+                type: String,
+            },
+            version: {
+                description: "Package version to get changelog of. Omit to use candidate version.",
+                type: String,
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        schema: UPID_SCHEMA,
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Retrieve the changelog of the specified package.
+fn apt_get_changelog(
+    param: Value,
+) -> Result<Value, Error> {
+
+    let name = crate::tools::required_string_param(&param, "name")?.to_owned();
+    let version = param["version"].as_str();
+
+    let pkg_info = apt::list_installed_apt_packages(|data| {
+        match version {
+            Some(version) => version == data.active_version,
+            None => data.active_version == data.candidate_version
+        }
+    }, Some(&name));
+
+    if pkg_info.len() == 0 {
+        bail!("Package '{}' not found", name);
+    }
+
+    let changelog_url = &pkg_info[0].change_log_url;
+    // FIXME: use 'apt-get changelog' for proxmox packages as well, once repo supports it
+    if changelog_url.starts_with("http://download.proxmox.com/") {
+        let changelog = crate::tools::runtime::block_on(http::get_string(changelog_url))
+            .map_err(|err| format_err!("Error downloading changelog from '{}': {}", changelog_url, err))?;
+        return Ok(json!(changelog));
+    } else {
+        let mut command = std::process::Command::new("apt-get");
+        command.arg("changelog");
+        command.arg("-qq"); // don't display download progress
+        command.arg(name);
+        let output = crate::tools::run_command(command, None)?;
+        return Ok(json!(output));
+    }
+}
+
 const SUBDIRS: SubdirMap = &[
+    ("changelog", &Router::new().get(&API_METHOD_APT_GET_CHANGELOG)),
     ("update", &Router::new()
         .get(&API_METHOD_APT_UPDATE_AVAILABLE)
         .post(&API_METHOD_APT_UPDATE_DATABASE)

src/api2/node/disks.rs

@@ -13,7 +13,7 @@ use crate::tools::disks::{
 };
 use crate::server::WorkerTask;
 
-use crate::api2::types::{Userid, UPID_SCHEMA, NODE_SCHEMA, BLOCKDEVICE_NAME_SCHEMA};
+use crate::api2::types::{Authid, UPID_SCHEMA, NODE_SCHEMA, BLOCKDEVICE_NAME_SCHEMA};
 
 pub mod directory;
 pub mod zfs;
@@ -140,7 +140,7 @@ pub fn initialize_disk(
 
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     let info = get_disk_usage_info(&disk, true)?;
 
@@ -149,7 +149,7 @@ pub fn initialize_disk(
     }
 
     let upid_str = WorkerTask::new_thread(
-        "diskinit", Some(disk.clone()), userid, to_stdout, move |worker|
+        "diskinit", Some(disk.clone()), auth_id, to_stdout, move |worker|
         {
             worker.log(format!("initialize disk {}", disk));
 

src/api2/node/disks/directory.rs

@@ -134,7 +134,7 @@ pub fn create_datastore_disk(
 
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     let info = get_disk_usage_info(&disk, true)?;
 
@@ -143,7 +143,7 @@ pub fn create_datastore_disk(
     }
 
     let upid_str = WorkerTask::new_thread(
-        "dircreate", Some(name.clone()), userid, to_stdout, move |worker|
+        "dircreate", Some(name.clone()), auth_id, to_stdout, move |worker|
         {
             worker.log(format!("create datastore '{}' on disk {}", name, disk));
 

src/api2/node/disks/zfs.rs

@@ -1,6 +1,6 @@
 use anyhow::{bail, Error};
 use serde_json::{json, Value};
-use ::serde::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize};
 
 use proxmox::api::{
     api, Permission, RpcEnvironment, RpcEnvironmentType,
@@ -256,7 +256,7 @@ pub fn create_zpool(
 
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     let add_datastore = add_datastore.unwrap_or(false);
 
@@ -316,7 +316,7 @@ pub fn create_zpool(
     }
 
      let upid_str = WorkerTask::new_thread(
-        "zfscreate", Some(name.clone()), userid, to_stdout, move |worker|
+        "zfscreate", Some(name.clone()), auth_id, to_stdout, move |worker|
         {
             worker.log(format!("create {:?} zpool '{}' on devices '{}'", raidlevel, name, devices_text));
 

src/api2/node/network.rs

@@ -684,9 +684,9 @@ pub async fn reload_network_config(
 
     network::assert_ifupdown2_installed()?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
-    let upid_str = WorkerTask::spawn("srvreload", Some(String::from("networking")), userid, true, |_worker| async {
+    let upid_str = WorkerTask::spawn("srvreload", Some(String::from("networking")), auth_id, true, |_worker| async {
 
         let _ = std::fs::rename(network::NETWORK_INTERFACES_NEW_FILENAME, network::NETWORK_INTERFACES_FILENAME);
 

src/api2/node/report.rs

@@ -0,0 +1,35 @@
+use anyhow::Error;
+use proxmox::api::{api, ApiMethod, Permission, Router, RpcEnvironment};
+use serde_json::{json, Value};
+
+use crate::api2::types::*;
+use crate::config::acl::PRIV_SYS_AUDIT;
+use crate::server::generate_report;
+
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+        },
+    },
+    returns: {
+        type: String,
+        description: "Returns report of the node"
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "status"], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// Generate a report
+fn get_report(
+    _param: Value,
+    _info: &ApiMethod,
+    _rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Value, Error> {
+    Ok(json!(generate_report()))
+}
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_GET_REPORT);

src/api2/node/rrd.rs

@@ -31,10 +31,8 @@ pub fn create_value_from_rrd(
                 } else {
                     result.push(json!({ "time": t }));
                 }
-            } else {
-                if let Some(value) = list[index] {
-                    result[index][name] = value.into();
-                }
+            } else if let Some(value) = list[index] {
+                result[index][name] = value.into();
             }
             t += reso;
         }

src/api2/node/services.rs

@@ -182,7 +182,7 @@ fn get_service_state(
     Ok(json_service_state(&service, status))
 }
 
-fn run_service_command(service: &str, cmd: &str, userid: Userid) -> Result<Value, Error> {
+fn run_service_command(service: &str, cmd: &str, auth_id: Authid) -> Result<Value, Error> {
 
     let workerid = format!("srv{}", &cmd);
 
@@ -196,7 +196,7 @@ fn run_service_command(service: &str, cmd: &str, userid: Userid) -> Result<Value
     let upid = WorkerTask::new_thread(
         &workerid,
         Some(service.clone()),
-        userid,
+        auth_id,
         false,
         move |_worker| {
 
@@ -244,11 +244,11 @@ fn start_service(
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     log::info!("starting service {}", service);
 
-    run_service_command(&service, "start", userid)
+    run_service_command(&service, "start", auth_id)
 }
 
 #[api(
@@ -274,11 +274,11 @@ fn stop_service(
     rpcenv: &mut dyn RpcEnvironment,
  ) -> Result<Value, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     log::info!("stopping service {}", service);
 
-    run_service_command(&service, "stop", userid)
+    run_service_command(&service, "stop", auth_id)
 }
 
 #[api(
@@ -304,15 +304,15 @@ fn restart_service(
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     log::info!("re-starting service {}", service);
 
     if &service == "proxmox-backup-proxy" {
         // special case, avoid aborting running tasks
-        run_service_command(&service, "reload", userid)
+        run_service_command(&service, "reload", auth_id)
     } else {
-        run_service_command(&service, "restart", userid)
+        run_service_command(&service, "restart", auth_id)
     }
 }
 
@@ -339,11 +339,11 @@ fn reload_service(
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
     log::info!("reloading service {}", service);
 
-    run_service_command(&service, "reload", userid)
+    run_service_command(&service, "reload", auth_id)
 }
 
 

src/api2/node/subscription.rs

@@ -1,12 +1,69 @@
-use anyhow::{Error};
-use serde_json::{json, Value};
+use anyhow::{Error, format_err, bail};
+use serde_json::Value;
 
 use proxmox::api::{api, Router, RpcEnvironment, Permission};
 
 use crate::tools;
-use crate::config::acl::PRIV_SYS_AUDIT;
+use crate::tools::subscription::{self, SubscriptionStatus, SubscriptionInfo};
+use crate::config::acl::{PRIV_SYS_AUDIT,PRIV_SYS_MODIFY};
 use crate::config::cached_user_info::CachedUserInfo;
-use crate::api2::types::{NODE_SCHEMA, Userid};
+use crate::api2::types::{NODE_SCHEMA, SUBSCRIPTION_KEY_SCHEMA, Authid};
+
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            force: {
+                description: "Always connect to server, even if information in cache is up to date.",
+                type: bool,
+                optional: true,
+                default: false,
+            },
+        },
+    },
+    protected: true,
+    access: {
+        permission: &Permission::Privilege(&["system"], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Check and update subscription status.
+pub fn check_subscription(
+    force: bool,
+) -> Result<(), Error> {
+    // FIXME: drop once proxmox-api-macro is bumped to >> 5.0.0-1
+    let _remove_me = API_METHOD_CHECK_SUBSCRIPTION_PARAM_DEFAULT_FORCE;
+
+    let info = match subscription::read_subscription() {
+        Err(err) => bail!("could not read subscription status: {}", err),
+        Ok(Some(info)) => info,
+        Ok(None) => return Ok(()),
+    };
+
+    let server_id = tools::get_hardware_address()?;
+    let key = if let Some(key) = info.key {
+        // always update apt auth if we have a key to ensure user can access enterprise repo
+        subscription::update_apt_auth(Some(key.to_owned()), Some(server_id.to_owned()))?;
+        key
+    } else {
+        String::new()
+    };
+
+    if !force && info.status == SubscriptionStatus::ACTIVE {
+        let age = proxmox::tools::time::epoch_i64() - info.checktime.unwrap_or(i64::MAX);
+        if age < subscription::MAX_LOCAL_KEY_AGE {
+            return Ok(());
+        }
+    }
+
+    let info = subscription::check_subscription(key, server_id)?;
+
+    subscription::write_subscription(info)
+        .map_err(|e| format_err!("Error writing updated subscription status - {}", e))?;
+
+    Ok(())
+}
 
 #[api(
     input: {
@@ -18,51 +75,103 @@ use crate::api2::types::{NODE_SCHEMA, Userid};
     },
     returns: {
         description: "Subscription status.",
-        properties: {
-            status: {
-                type: String,
-                description: "'NotFound', 'active' or 'inactive'."
-            },
-            message: {
-                type: String,
-                description: "Human readable problem description.",
-            },
-            serverid: {
-                type: String,
-                description: "The unique server ID, if permitted to access.",
-            },
-            url: {
-                type: String,
-                description: "URL to Web Shop.",
-            },
-        },
+        type: SubscriptionInfo,
     },
     access: {
         permission: &Permission::Anybody,
     },
 )]
 /// Read subscription info.
-fn get_subscription(
+pub fn get_subscription(
     _param: Value,
     rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-    let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &[]);
-    let server_id = if (user_privs & PRIV_SYS_AUDIT) != 0 {
-        tools::get_hardware_address()?
-    } else {
-        "hidden".to_string()
+) -> Result<SubscriptionInfo, Error> {
+    let url = "https://www.proxmox.com/en/proxmox-backup-server/pricing";
+
+    let info = match subscription::read_subscription() {
+        Err(err) => bail!("could not read subscription status: {}", err),
+        Ok(Some(info)) => info,
+        Ok(None) => SubscriptionInfo {
+            status: SubscriptionStatus::NOTFOUND,
+            message: Some("There is no subscription key".into()),
+            serverid: Some(tools::get_hardware_address()?),
+            url:  Some(url.into()),
+            ..Default::default()
+        },
     };
 
-    let url = "https://www.proxmox.com/en/proxmox-backup-server/pricing";
-    Ok(json!({
-        "status": "NotFound",
-        "message": "There is no subscription key",
-        "serverid": server_id,
-        "url":  url,
-     }))
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    let user_privs = user_info.lookup_privs(&auth_id, &[]);
+
+    if (user_privs & PRIV_SYS_AUDIT) == 0 {
+        // not enough privileges for full state
+        return Ok(SubscriptionInfo {
+            status: info.status,
+            message: info.message,
+            url: info.url,
+            ..Default::default()
+        });
+    };
+
+    Ok(info)
+}
+
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            key: {
+                schema: SUBSCRIPTION_KEY_SCHEMA,
+            },
+        },
+    },
+    protected: true,
+    access: {
+        permission: &Permission::Privilege(&["system"], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Set a subscription key and check it.
+pub fn set_subscription(
+    key: String,
+) -> Result<(), Error> {
+
+    let server_id = tools::get_hardware_address()?;
+
+    let info = subscription::check_subscription(key, server_id.to_owned())?;
+
+    subscription::write_subscription(info)
+        .map_err(|e| format_err!("Error writing subscription status - {}", e))?;
+
+    Ok(())
+}
+
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+        },
+    },
+    protected: true,
+    access: {
+        permission: &Permission::Privilege(&["system"], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Delete subscription info.
+pub fn delete_subscription() -> Result<(), Error> {
+
+    subscription::delete_subscription()
+        .map_err(|err| format_err!("Deleting subscription failed: {}", err))?;
+
+    Ok(())
 }
 
 pub const ROUTER: Router = Router::new()
+    .post(&API_METHOD_CHECK_SUBSCRIPTION)
+    .put(&API_METHOD_SET_SUBSCRIPTION)
+    .delete(&API_METHOD_DELETE_SUBSCRIPTION)
     .get(&API_METHOD_GET_SUBSCRIPTION);

src/api2/node/tasks.rs

@@ -1,7 +1,7 @@
 use std::fs::File;
 use std::io::{BufRead, BufReader};
 
-use anyhow::{Error};
+use anyhow::{bail, Error};
 use serde_json::{json, Value};
 
 use proxmox::api::{api, Router, RpcEnvironment, Permission};
@@ -9,11 +9,89 @@ use proxmox::api::router::SubdirMap;
 use proxmox::{identity, list_subdirs_api_method, sortable};
 
 use crate::tools;
+
 use crate::api2::types::*;
+use crate::api2::pull::check_pull_privs;
+
 use crate::server::{self, UPID, TaskState, TaskListInfoIterator};
-use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
+use crate::config::acl::{
+    PRIV_DATASTORE_MODIFY,
+    PRIV_DATASTORE_VERIFY,
+    PRIV_SYS_AUDIT,
+    PRIV_SYS_MODIFY,
+};
 use crate::config::cached_user_info::CachedUserInfo;
 
+// matches respective job execution privileges
+fn check_job_privs(auth_id: &Authid, user_info: &CachedUserInfo, upid: &UPID) -> Result<(), Error> {
+    match (upid.worker_type.as_str(), &upid.worker_id) {
+        ("verificationjob", Some(workerid)) => {
+            if let Some(captures) = VERIFICATION_JOB_WORKER_ID_REGEX.captures(&workerid) {
+                if let Some(store) = captures.get(1) {
+                    return user_info.check_privs(&auth_id,
+                                                 &["datastore", store.as_str()],
+                                                 PRIV_DATASTORE_VERIFY,
+                                                 true);
+                }
+            }
+        },
+        ("syncjob", Some(workerid)) => {
+            if let Some(captures) = SYNC_JOB_WORKER_ID_REGEX.captures(&workerid) {
+                let remote = captures.get(1);
+                let remote_store = captures.get(2);
+                let local_store = captures.get(3);
+
+                if let (Some(remote), Some(remote_store), Some(local_store)) =
+                    (remote, remote_store, local_store) {
+
+                    return check_pull_privs(&auth_id,
+                                            local_store.as_str(),
+                                            remote.as_str(),
+                                            remote_store.as_str(),
+                                            false);
+                }
+            }
+        },
+        ("garbage_collection", Some(workerid)) => {
+            return user_info.check_privs(&auth_id,
+                                         &["datastore", &workerid],
+                                         PRIV_DATASTORE_MODIFY,
+                                         true)
+        },
+        ("prune", Some(workerid)) => {
+            return user_info.check_privs(&auth_id,
+                                         &["datastore",
+                                         &workerid],
+                                         PRIV_DATASTORE_MODIFY,
+                                         true);
+        },
+        _ => bail!("not a scheduled job task"),
+    };
+
+    bail!("not a scheduled job task");
+}
+
+fn check_task_access(auth_id: &Authid, upid: &UPID) -> Result<(), Error> {
+    let task_auth_id = &upid.auth_id;
+    if auth_id == task_auth_id
+        || (task_auth_id.is_token() && &Authid::from(task_auth_id.user().clone()) == auth_id) {
+        // task owner can always read
+        Ok(())
+    } else {
+        let user_info = CachedUserInfo::new()?;
+
+        let task_privs = user_info.lookup_privs(auth_id, &["system", "tasks"]);
+        if task_privs & PRIV_SYS_AUDIT != 0 {
+            // allowed to read all tasks in general
+            Ok(())
+        } else if check_job_privs(&auth_id, &user_info, upid).is_ok() {
+            // job which the user/token could have configured/manually executed
+            Ok(())
+        } else {
+            bail!("task access not allowed");
+        }
+    }
+}
 
 #[api(
     input: {
@@ -27,7 +105,7 @@ use crate::config::cached_user_info::CachedUserInfo;
         },
     },
     returns: {
-        description: "Task status nformation.",
+        description: "Task status information.",
         properties: {
             node: {
                 schema: NODE_SCHEMA,
@@ -57,9 +135,13 @@ use crate::config::cached_user_info::CachedUserInfo;
                 description: "Worker ID (arbitrary ASCII string)",
             },
             user: {
-                type: String,
+                type: Userid,
                 description: "The user who started the task.",
             },
+            tokenid: {
+                type: Tokenname,
+                optional: true,
+            },
             status: {
                 type: String,
                 description: "'running' or 'stopped'",
@@ -72,7 +154,7 @@ use crate::config::cached_user_info::CachedUserInfo;
         },
     },
     access: {
-        description: "Users can access there own tasks, or need Sys.Audit on /system/tasks.",
+        description: "Users can access their own tasks, or need Sys.Audit on /system/tasks.",
         permission: &Permission::Anybody,
     },
 )]
@@ -84,12 +166,8 @@ async fn get_task_status(
 
     let upid = extract_upid(&param)?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-
-    if userid != upid.userid {
-        let user_info = CachedUserInfo::new()?;
-        user_info.check_privs(&userid, &["system", "tasks"], PRIV_SYS_AUDIT, false)?;
-    }
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    check_task_access(&auth_id, &upid)?;
 
     let mut result = json!({
         "upid": param["upid"],
@@ -99,9 +177,13 @@ async fn get_task_status(
         "starttime": upid.starttime,
         "type": upid.worker_type,
         "id": upid.worker_id,
-        "user": upid.userid,
+        "user": upid.auth_id.user(),
     });
 
+    if upid.auth_id.is_token() {
+        result["tokenid"] = Value::from(upid.auth_id.tokenname().unwrap().as_str());
+    }
+
     if crate::server::worker_is_active(&upid).await? {
         result["status"] = Value::from("running");
     } else {
@@ -161,12 +243,9 @@ async fn read_task_log(
 
     let upid = extract_upid(&param)?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
-    if userid != upid.userid {
-        let user_info = CachedUserInfo::new()?;
-        user_info.check_privs(&userid, &["system", "tasks"], PRIV_SYS_AUDIT, false)?;
-    }
+    check_task_access(&auth_id, &upid)?;
 
     let test_status = param["test-status"].as_bool().unwrap_or(false);
 
@@ -234,11 +313,11 @@ fn stop_task(
 
     let upid = extract_upid(&param)?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
-    if userid != upid.userid {
+    if auth_id != upid.auth_id {
         let user_info = CachedUserInfo::new()?;
-        user_info.check_privs(&userid, &["system", "tasks"], PRIV_SYS_MODIFY, false)?;
+        user_info.check_privs(&auth_id, &["system", "tasks"], PRIV_SYS_MODIFY, false)?;
     }
 
     server::abort_worker_async(upid);
@@ -260,7 +339,7 @@ fn stop_task(
             },
             limit: {
                 type: u64,
-                description: "Only list this amount of tasks.",
+                description: "Only list this amount of tasks. (0 means no limit)",
                 default: 50,
                 optional: true,
             },
@@ -285,6 +364,29 @@ fn stop_task(
                 type: String,
                 description: "Only list tasks from this user.",
             },
+            since: {
+                type: i64,
+                description: "Only list tasks since this UNIX epoch.",
+                optional: true,
+            },
+            until: {
+                type: i64,
+                description: "Only list tasks until this UNIX epoch.",
+                optional: true,
+            },
+            typefilter: {
+                optional: true,
+                type: String,
+                description: "Only list tasks whose type contains this.",
+            },
+            statusfilter: {
+                optional: true,
+                type: Array,
+                description: "Only list tasks which have any one of the listed status.",
+                items: {
+                    type: TaskStateType,
+                },
+            },
         },
     },
     returns: {
@@ -304,32 +406,52 @@ pub fn list_tasks(
     errors: bool,
     running: bool,
     userfilter: Option<String>,
+    since: Option<i64>,
+    until: Option<i64>,
+    typefilter: Option<String>,
+    statusfilter: Option<Vec<TaskStateType>>,
     param: Value,
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<TaskListItem>, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &["system", "tasks"]);
+    let user_privs = user_info.lookup_privs(&auth_id, &["system", "tasks"]);
 
     let list_all = (user_privs & PRIV_SYS_AUDIT) != 0;
 
     let store = param["store"].as_str();
 
     let list = TaskListInfoIterator::new(running)?;
+    let limit = if limit > 0 { limit as usize } else { usize::MAX };
 
     let result: Vec<TaskListItem> = list
-        .take_while(|info| !info.is_err())
+        .skip_while(|info| {
+            match (info, until) {
+                (Ok(info), Some(until)) => info.upid.starttime > until,
+                (Ok(_), None) => false,
+                (Err(_), _) => false,
+            }
+        })
+        .take_while(|info| {
+            match (info, since) {
+                (Ok(info), Some(since)) => info.upid.starttime > since,
+                (Ok(_), None) => true,
+                (Err(_), _) => false,
+            }
+        })
         .filter_map(|info| {
         let info = match info {
             Ok(info) => info,
             Err(_) => return None,
         };
 
-        if !list_all && info.upid.userid != userid { return None; }
+        if !list_all && check_task_access(&auth_id, &info.upid).is_err() {
+            return None;
+        }
 
-        if let Some(userid) = &userfilter {
-            if !info.upid.userid.as_str().contains(userid) { return None; }
+        if let Some(needle) = &userfilter {
+            if !info.upid.auth_id.to_string().contains(needle) { return None; }
         }
 
         if let Some(store) = store {
@@ -342,7 +464,7 @@ pub fn list_tasks(
             if info.upid.worker_type == "backup" || info.upid.worker_type == "restore" ||
                 info.upid.worker_type == "prune"
             {
-                let prefix = format!("{}_", store);
+                let prefix = format!("{}:", store);
                 if !worker_id.starts_with(&prefix) { return None; }
             } else if info.upid.worker_type == "garbage_collection" {
                 if worker_id != store { return None; }
@@ -351,19 +473,31 @@ pub fn list_tasks(
             }
         }
 
-        match info.state {
-            Some(_) if running => return None,
-            Some(crate::server::TaskState::OK { .. }) if errors => return None,
+        if let Some(typefilter) = &typefilter {
+            if !info.upid.worker_type.contains(typefilter) {
+                return None;
+            }
+        }
+
+        match (&info.state, &statusfilter) {
+            (Some(_), _) if running => return None,
+            (Some(crate::server::TaskState::OK { .. }), _) if errors => return None,
+            (Some(state), Some(filters)) => {
+                if !filters.contains(&state.tasktype()) {
+                    return None;
+                }
+            },
+            (None, Some(_)) => return None,
             _ => {},
         }
 
         Some(info.into())
     }).skip(start as usize)
-        .take(limit as usize)
+        .take(limit)
         .collect();
 
     let mut count = result.len() + start as usize;
-    if result.len() > 0 && result.len() >= limit as usize { // we have a 'virtual' entry as long as we have any new
+    if result.len() > 0 && result.len() >= limit { // we have a 'virtual' entry as long as we have any new
         count += 1;
     }
 

src/api2/ping.rs

@@ -0,0 +1,29 @@
+use anyhow::{Error};
+use serde_json::{json, Value};
+
+use proxmox::api::{api, Router, Permission};
+
+#[api(
+    returns: {
+        description: "Dummy ping",
+        type: Object,
+        properties: {
+            pong: {
+                description: "Always true",
+                type: bool,
+            }
+        }
+    },
+    access: {
+        description: "Anyone can access this, because it's used for a cheap check if the API daemon is online.",
+        permission: &Permission::World,
+    }
+)]
+/// Dummy method which replies with `{ "pong": True }`
+fn ping() -> Result<Value, Error> {
+    Ok(json!({
+        "pong": true,
+    }))
+}
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_PING);

src/api2/pull.rs

@@ -7,21 +7,20 @@ use futures::{select, future::FutureExt};
 use proxmox::api::api;
 use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
 
-use crate::server::{WorkerTask};
+use crate::server::{WorkerTask, jobstate::Job};
 use crate::backup::DataStore;
-use crate::client::{HttpClient, HttpClientOptions, BackupRepository, pull::pull_store};
+use crate::client::{HttpClient, BackupRepository, pull::pull_store};
 use crate::api2::types::*;
 use crate::config::{
     remote,
     sync::SyncJobConfig,
-    jobstate::Job,
     acl::{PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ},
     cached_user_info::CachedUserInfo,
 };
 
 
 pub fn check_pull_privs(
-    userid: &Userid,
+    auth_id: &Authid,
     store: &str,
     remote: &str,
     remote_store: &str,
@@ -30,11 +29,11 @@ pub fn check_pull_privs(
 
     let user_info = CachedUserInfo::new()?;
 
-    user_info.check_privs(userid, &["datastore", store], PRIV_DATASTORE_BACKUP, false)?;
-    user_info.check_privs(userid, &["remote", remote, remote_store], PRIV_REMOTE_READ, false)?;
+    user_info.check_privs(auth_id, &["datastore", store], PRIV_DATASTORE_BACKUP, false)?;
+    user_info.check_privs(auth_id, &["remote", remote, remote_store], PRIV_REMOTE_READ, false)?;
 
     if delete {
-        user_info.check_privs(userid, &["datastore", store], PRIV_DATASTORE_PRUNE, false)?;
+        user_info.check_privs(auth_id, &["datastore", store], PRIV_DATASTORE_PRUNE, false)?;
     }
 
     Ok(())
@@ -51,17 +50,9 @@ pub async fn get_pull_parameters(
     let (remote_config, _digest) = remote::config()?;
     let remote: remote::Remote = remote_config.lookup("remote", remote)?;
 
-    let options = HttpClientOptions::new()
-        .password(Some(remote.password.clone()))
-        .fingerprint(remote.fingerprint.clone());
-
     let src_repo = BackupRepository::new(Some(remote.userid.clone()), Some(remote.host.clone()), remote.port, remote_store.to_string());
 
-    let client = HttpClient::new(&src_repo.host(), src_repo.port(), &src_repo.user(), options)?;
-    let _auth_info = client.login() // make sure we can auth
-        .await
-        .map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?;
-
+    let client = crate::api2::config::remote::remote_client(remote).await?;
 
     Ok((client, src_repo, tgt_store))
 }
@@ -69,27 +60,35 @@ pub async fn get_pull_parameters(
 pub fn do_sync_job(
     mut job: Job,
     sync_job: SyncJobConfig,
-    userid: &Userid,
+    auth_id: &Authid,
     schedule: Option<String>,
 ) -> Result<String, Error> {
 
-    let job_id = job.jobname().to_string();
+    let job_id = format!("{}:{}:{}:{}",
+                         sync_job.remote,
+                         sync_job.remote_store,
+                         sync_job.store,
+                         job.jobname());
     let worker_type = job.jobtype().to_string();
 
+    let (email, notify) = crate::server::lookup_datastore_notify_settings(&sync_job.store);
+
     let upid_str = WorkerTask::spawn(
         &worker_type,
-        Some(job.jobname().to_string()),
-        userid.clone(),
+        Some(job_id.clone()),
+        auth_id.clone(),
         false,
         move |worker| async move {
 
             job.start(&worker.upid().to_string())?;
 
             let worker2 = worker.clone();
+            let sync_job2 = sync_job.clone();
 
             let worker_future = async move {
 
                 let delete = sync_job.remove_vanished.unwrap_or(true);
+                let sync_owner = sync_job.owner.unwrap_or(Authid::root_auth_id().clone());
                 let (client, src_repo, tgt_store) = get_pull_parameters(&sync_job.store, &sync_job.remote, &sync_job.remote_store).await?;
 
                 worker.log(format!("Starting datastore sync job '{}'", job_id));
@@ -99,7 +98,7 @@ pub fn do_sync_job(
                 worker.log(format!("Sync datastore '{}' from '{}/{}'",
                         sync_job.store, sync_job.remote, sync_job.remote_store));
 
-                crate::client::pull::pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, Userid::backup_userid().clone()).await?;
+                crate::client::pull::pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, sync_owner).await?;
 
                 worker.log(format!("sync job '{}' end", &job_id));
 
@@ -108,12 +107,12 @@ pub fn do_sync_job(
 
             let mut abort_future = worker2.abort_future().map(|_| Err(format_err!("sync aborted")));
 
-            let res = select!{
+            let result = select!{
                 worker = worker_future.fuse() => worker,
                 abort = abort_future => abort,
             };
 
-            let status = worker2.create_state(&res);
+            let status = worker2.create_state(&result);
 
             match job.finish(status) {
                 Ok(_) => {},
@@ -122,7 +121,13 @@ pub fn do_sync_job(
                 }
             }
 
-            res
+            if let Some(email) = email {
+                if let Err(err) = crate::server::send_sync_status(&email, notify, &sync_job2, &result) {
+                    eprintln!("send sync notification failed: {}", err);
+                }
+            }
+
+            result
         })?;
 
     Ok(upid_str)
@@ -165,19 +170,19 @@ async fn pull (
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<String, Error> {
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let delete = remove_vanished.unwrap_or(true);
 
-    check_pull_privs(&userid, &store, &remote, &remote_store, delete)?;
+    check_pull_privs(&auth_id, &store, &remote, &remote_store, delete)?;
 
     let (client, src_repo, tgt_store) = get_pull_parameters(&store, &remote, &remote_store).await?;
 
     // fixme: set to_stdout to false?
-    let upid_str = WorkerTask::spawn("sync", Some(store.clone()), userid.clone(), true, move |worker| async move {
+    let upid_str = WorkerTask::spawn("sync", Some(store.clone()), auth_id.clone(), true, move |worker| async move {
 
         worker.log(format!("sync datastore '{}' start", store));
 
-        let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, userid);
+        let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, auth_id);
         let future = select!{
             success = pull_future.fuse() => success,
             abort = worker.abort_future().map(|_| Err(format_err!("pull aborted"))) => abort,

src/api2/reader.rs

@@ -17,6 +17,7 @@ use crate::tools;
 use crate::config::acl::{PRIV_DATASTORE_READ, PRIV_DATASTORE_BACKUP};
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::api2::helpers;
+use crate::tools::fs::lock_dir_noblock_shared;
 
 mod environment;
 use environment::*;
@@ -54,11 +55,11 @@ fn upgrade_to_backup_reader_protocol(
     async move {
         let debug = param["debug"].as_bool().unwrap_or(false);
 
-        let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+        let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
         let store = tools::required_string_param(&param, "store")?.to_owned();
 
         let user_info = CachedUserInfo::new()?;
-        let privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+        let privs = user_info.lookup_privs(&auth_id, &["datastore", &store]);
 
         let priv_read = privs & PRIV_DATASTORE_READ != 0;
         let priv_backup = privs & PRIV_DATASTORE_BACKUP != 0;
@@ -93,21 +94,29 @@ fn upgrade_to_backup_reader_protocol(
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
         if !priv_read {
             let owner = datastore.get_owner(backup_dir.group())?;
-            if owner != userid {
+            let correct_owner = owner == auth_id
+                || (owner.is_token()
+                    && Authid::from(owner.user().clone()) == auth_id);
+            if !correct_owner {
                 bail!("backup owner check failed!");
             }
         }
 
+        let _guard = lock_dir_noblock_shared(
+            &datastore.snapshot_path(&backup_dir),
+            "snapshot",
+            "locked by another operation")?;
+
         let path = datastore.base_path();
 
         //let files = BackupInfo::list_files(&path, &backup_dir)?;
 
-        let worker_id = format!("{}_{}_{}_{:08X}", store, backup_type, backup_id, backup_dir.backup_time());
+        let worker_id = format!("{}:{}/{}/{:08X}", store, backup_type, backup_id, backup_dir.backup_time());
 
-        WorkerTask::spawn("reader", Some(worker_id), userid.clone(), true, move |worker| {
+        WorkerTask::spawn("reader", Some(worker_id), auth_id.clone(), true, move |worker| {
             let mut env = ReaderEnvironment::new(
                 env_type,
-                userid,
+                auth_id,
                 worker.clone(),
                 datastore,
                 backup_dir,
@@ -146,11 +155,14 @@ fn upgrade_to_backup_reader_protocol(
 
             use futures::future::Either;
             futures::future::select(req_fut, abort_future)
-                .map(|res| match res {
-                    Either::Left((Ok(res), _)) => Ok(res),
-                    Either::Left((Err(err), _)) => Err(err),
-                    Either::Right((Ok(res), _)) => Ok(res),
-                    Either::Right((Err(err), _)) => Err(err),
+                .map(move |res| {
+                    let _guard = _guard;
+                    match res {
+                        Either::Left((Ok(res), _)) => Ok(res),
+                        Either::Left((Err(err), _)) => Err(err),
+                        Either::Right((Ok(res), _)) => Ok(res),
+                        Either::Right((Err(err), _)) => Err(err),
+                    }
                 })
                 .map_ok(move |_| env.log("reader finished successfully"))
         })?;

src/api2/reader/environment.rs

@@ -5,7 +5,7 @@ use serde_json::{json, Value};
 
 use proxmox::api::{RpcEnvironment, RpcEnvironmentType};
 
-use crate::api2::types::Userid;
+use crate::api2::types::Authid;
 use crate::backup::*;
 use crate::server::formatter::*;
 use crate::server::WorkerTask;
@@ -17,7 +17,7 @@ use crate::server::WorkerTask;
 pub struct ReaderEnvironment {
     env_type: RpcEnvironmentType,
     result_attributes: Value,
-    user: Userid,
+    auth_id: Authid,
     pub debug: bool,
     pub formatter: &'static OutputFormatter,
     pub worker: Arc<WorkerTask>,
@@ -29,7 +29,7 @@ pub struct ReaderEnvironment {
 impl ReaderEnvironment {
     pub fn new(
         env_type: RpcEnvironmentType,
-        user: Userid,
+        auth_id: Authid,
         worker: Arc<WorkerTask>,
         datastore: Arc<DataStore>,
         backup_dir: BackupDir,
@@ -39,7 +39,7 @@ impl ReaderEnvironment {
         Self {
             result_attributes: json!({}),
             env_type,
-            user,
+            auth_id,
             worker,
             datastore,
             debug: false,
@@ -82,12 +82,12 @@ impl RpcEnvironment for ReaderEnvironment {
         self.env_type
     }
 
-    fn set_user(&mut self, _user: Option<String>) {
-        panic!("unable to change user");
+    fn set_auth_id(&mut self, _auth_id: Option<String>) {
+        panic!("unable to change auth_id");
     }
 
-    fn get_user(&self) -> Option<String> {
-        Some(self.user.to_string())
+    fn get_auth_id(&self) -> Option<String> {
+        Some(self.auth_id.to_string())
     }
 }
 

src/api2/status.rs

@@ -16,18 +16,14 @@ use crate::api2::types::{
     DATASTORE_SCHEMA,
     RRDMode,
     RRDTimeFrameResolution,
-    TaskListItem,
-    TaskStateType,
-    Userid,
+    Authid,
 };
 
-use crate::server;
 use crate::backup::{DataStore};
 use crate::config::datastore;
 use crate::tools::statistics::{linear_regression};
 use crate::config::cached_user_info::CachedUserInfo;
 use crate::config::acl::{
-    PRIV_SYS_AUDIT,
     PRIV_DATASTORE_AUDIT,
     PRIV_DATASTORE_BACKUP,
 };
@@ -87,13 +83,13 @@ fn datastore_status(
 
     let (config, _digest) = datastore::config()?;
 
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let user_info = CachedUserInfo::new()?;
 
     let mut list = Vec::new();
 
     for (store, (_, _)) in &config.sections {
-        let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+        let user_privs = user_info.lookup_privs(&auth_id, &["datastore", &store]);
         let allowed = (user_privs & (PRIV_DATASTORE_AUDIT| PRIV_DATASTORE_BACKUP)) != 0;
         if !allowed {
             continue;
@@ -179,103 +175,8 @@ fn datastore_status(
     Ok(list.into())
 }
 
-#[api(
-    input: {
-        properties: {
-            since: {
-                type: i64,
-                description: "Only list tasks since this UNIX epoch.",
-                optional: true,
-            },
-            typefilter: {
-                optional: true,
-                type: String,
-                description: "Only list tasks, whose type contains this string.",
-            },
-            statusfilter: {
-                optional: true,
-                type: Array,
-                description: "Only list tasks which have any one of the listed status.",
-                items: {
-                    type: TaskStateType,
-                },
-            },
-        },
-    },
-    returns: {
-        description: "A list of tasks.",
-        type: Array,
-        items: { type: TaskListItem },
-    },
-    access: {
-        description: "Users can only see there own tasks, unless the have Sys.Audit on /system/tasks.",
-        permission: &Permission::Anybody,
-    },
-)]
-/// List tasks.
-pub fn list_tasks(
-    since: Option<i64>,
-    typefilter: Option<String>,
-    statusfilter: Option<Vec<TaskStateType>>,
-    _param: Value,
-    rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Vec<TaskListItem>, Error> {
-
-    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
-    let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&userid, &["system", "tasks"]);
-
-    let list_all = (user_privs & PRIV_SYS_AUDIT) != 0;
-    let since = since.unwrap_or_else(|| 0);
-
-    let list: Vec<TaskListItem> = server::TaskListInfoIterator::new(false)?
-        .take_while(|info| {
-            match info {
-                Ok(info) => info.upid.starttime > since,
-                Err(_) => false
-            }
-        })
-        .filter_map(|info| {
-            match info {
-                Ok(info) => {
-                    if list_all || info.upid.userid == userid {
-                        if let Some(filter) = &typefilter {
-                            if !info.upid.worker_type.contains(filter) {
-                                return None;
-                            }
-                        }
-
-                        if let Some(filters) = &statusfilter {
-                            if let Some(state) = &info.state {
-                                let statetype = match state {
-                                    server::TaskState::OK { .. } => TaskStateType::OK,
-                                    server::TaskState::Unknown { .. } => TaskStateType::Unknown,
-                                    server::TaskState::Error { .. } => TaskStateType::Error,
-                                    server::TaskState::Warning { .. } => TaskStateType::Warning,
-                                };
-
-                                if !filters.contains(&statetype) {
-                                    return None;
-                                }
-                            }
-                        }
-
-                        Some(Ok(TaskListItem::from(info)))
-                    } else {
-                        None
-                    }
-                }
-                Err(err) => Some(Err(err))
-            }
-        })
-        .collect::<Result<Vec<TaskListItem>, Error>>()?;
-
-    Ok(list.into())
-}
-
 const SUBDIRS: SubdirMap = &[
     ("datastore-usage", &Router::new().get(&API_METHOD_DATASTORE_STATUS)),
-    ("tasks", &Router::new().get(&API_METHOD_LIST_TASKS)),
 ];
 
 pub const ROUTER: Router = Router::new()

src/api2/types/mod.rs

@@ -5,7 +5,7 @@ use proxmox::api::{api, schema::*};
 use proxmox::const_regex;
 use proxmox::{IPRE, IPRE_BRACKET, IPV4RE, IPV6RE, IPV4OCTET, IPV6H16, IPV6LS32};
 
-use crate::backup::CryptMode;
+use crate::backup::{CryptMode, BACKUP_ID_REGEX};
 use crate::server::UPID;
 
 #[macro_use]
@@ -14,9 +14,11 @@ mod macros;
 #[macro_use]
 mod userid;
 pub use userid::{Realm, RealmRef};
+pub use userid::{Tokenname, TokennameRef};
 pub use userid::{Username, UsernameRef};
 pub use userid::Userid;
-pub use userid::PROXMOX_GROUP_ID_SCHEMA;
+pub use userid::Authid;
+pub use userid::{PROXMOX_TOKEN_ID_SCHEMA, PROXMOX_TOKEN_NAME_SCHEMA, PROXMOX_GROUP_ID_SCHEMA};
 
 // File names: may not contain slashes, may not start with "."
 pub const FILENAME_FORMAT: ApiStringFormat = ApiStringFormat::VerifyFn(|name| {
@@ -57,6 +59,11 @@ const_regex!{
     /// any identifier command line tools work with.
     pub PROXMOX_SAFE_ID_REGEX = concat!(r"^", PROXMOX_SAFE_ID_REGEX_STR!(), r"$");
 
+    /// Regex for verification jobs 'DATASTORE:ACTUAL_JOB_ID'
+    pub VERIFICATION_JOB_WORKER_ID_REGEX = concat!(r"^(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):");
+    /// Regex for sync jobs 'REMOTE:REMOTE_DATASTORE:LOCAL_DATASTORE:ACTUAL_JOB_ID'
+    pub SYNC_JOB_WORKER_ID_REGEX = concat!(r"^(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):(", PROXMOX_SAFE_ID_REGEX_STR!(), r"):");
+
     pub SINGLE_LINE_COMMENT_REGEX = r"^[[:^cntrl:]]*$";
 
     pub HOSTNAME_REGEX = r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9])?)$";
@@ -65,12 +72,14 @@ const_regex!{
 
     pub DNS_NAME_OR_IP_REGEX = concat!(r"^(?:", DNS_NAME!(), "|",  IPRE!(), r")$");
 
-    pub BACKUP_REPO_URL_REGEX = concat!(r"^^(?:(?:(", USER_ID_REGEX_STR!(), ")@)?(", DNS_NAME!(), "|",  IPRE_BRACKET!() ,"):)?(?:([0-9]{1,5}):)?(", PROXMOX_SAFE_ID_REGEX_STR!(), r")$");
+    pub BACKUP_REPO_URL_REGEX = concat!(r"^^(?:(?:(", USER_ID_REGEX_STR!(), "|", APITOKEN_ID_REGEX_STR!(), ")@)?(", DNS_NAME!(), "|",  IPRE_BRACKET!() ,"):)?(?:([0-9]{1,5}):)?(", PROXMOX_SAFE_ID_REGEX_STR!(), r")$");
 
     pub CERT_FINGERPRINT_SHA256_REGEX = r"^(?:[0-9a-fA-F][0-9a-fA-F])(?::[0-9a-fA-F][0-9a-fA-F]){31}$";
 
     pub ACL_PATH_REGEX = concat!(r"^(?:/|", r"(?:/", PROXMOX_SAFE_ID_REGEX_STR!(), ")+", r")$");
 
+    pub SUBSCRIPTION_KEY_REGEX = concat!(r"^pbs(?:[cbsp])-[0-9a-f]{10}$");
+
     pub BLOCKDEVICE_NAME_REGEX = r"^(:?(:?h|s|x?v)d[a-z]+)|(:?nvme\d+n\d+)$";
 
     pub ZPOOL_NAME_REGEX = r"^[a-zA-Z][a-z0-9A-Z\-_.:]+$";
@@ -97,6 +106,9 @@ pub const CERT_FINGERPRINT_SHA256_FORMAT: ApiStringFormat =
 pub const PROXMOX_SAFE_ID_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&PROXMOX_SAFE_ID_REGEX);
 
+pub const BACKUP_ID_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&BACKUP_ID_REGEX);
+
 pub const SINGLE_LINE_COMMENT_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&SINGLE_LINE_COMMENT_REGEX);
 
@@ -127,6 +139,9 @@ pub const CIDR_V6_FORMAT: ApiStringFormat =
 pub const CIDR_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&CIDR_REGEX);
 
+pub const SUBSCRIPTION_KEY_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&SUBSCRIPTION_KEY_REGEX);
+
 pub const BLOCKDEVICE_NAME_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&BLOCKDEVICE_NAME_REGEX);
 
@@ -269,7 +284,7 @@ pub const BACKUP_TYPE_SCHEMA: Schema =
 
 pub const BACKUP_ID_SCHEMA: Schema =
     StringSchema::new("Backup ID.")
-    .format(&PROXMOX_SAFE_ID_FORMAT)
+    .format(&BACKUP_ID_FORMAT)
     .schema();
 
 pub const BACKUP_TIME_SCHEMA: Schema =
@@ -302,7 +317,7 @@ pub const PRUNE_SCHEDULE_SCHEMA: Schema = StringSchema::new(
     .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
     .schema();
 
-pub const VERIFY_SCHEDULE_SCHEMA: Schema = StringSchema::new(
+pub const VERIFICATION_SCHEDULE_SCHEMA: Schema = StringSchema::new(
     "Run verify job at specified schedule.")
     .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
     .schema();
@@ -324,6 +339,16 @@ pub const REMOVE_VANISHED_BACKUPS_SCHEMA: Schema = BooleanSchema::new(
     .default(true)
     .schema();
 
+pub const IGNORE_VERIFIED_BACKUPS_SCHEMA: Schema = BooleanSchema::new(
+    "Do not verify backups that are already verified if their verification is not outdated.")
+    .default(true)
+    .schema();
+
+pub const VERIFICATION_OUTDATED_AFTER_SCHEMA: Schema = IntegerSchema::new(
+    "Days after that a verification becomes outdated")
+    .minimum(1)
+    .schema();
+
 pub const SINGLE_LINE_COMMENT_SCHEMA: Schema = StringSchema::new("Comment (single line).")
     .format(&SINGLE_LINE_COMMENT_FORMAT)
     .schema();
@@ -336,6 +361,12 @@ pub const DNS_NAME_OR_IP_SCHEMA: Schema = StringSchema::new("DNS name or IP addr
     .format(&DNS_NAME_OR_IP_FORMAT)
     .schema();
 
+pub const SUBSCRIPTION_KEY_SCHEMA: Schema = StringSchema::new("Proxmox Backup Server subscription key.")
+    .format(&SUBSCRIPTION_KEY_FORMAT)
+    .min_length(15)
+    .max_length(16)
+    .schema();
+
 pub const BLOCKDEVICE_NAME_SCHEMA: Schema = StringSchema::new("Block device name (/sys/block/<name>).")
     .format(&BLOCKDEVICE_NAME_FORMAT)
     .min_length(3)
@@ -344,6 +375,25 @@ pub const BLOCKDEVICE_NAME_SCHEMA: Schema = StringSchema::new("Block device name
 
 // Complex type definitions
 
+#[api(
+    properties: {
+        store: {
+            schema: DATASTORE_SCHEMA,
+        },
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+    },
+)]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all="kebab-case")]
+/// Basic information about a datastore.
+pub struct DataStoreListItem {
+    pub store: String,
+    pub comment: Option<String>,
+}
+
 #[api(
     properties: {
         "backup-type": {
@@ -364,7 +414,7 @@ pub const BLOCKDEVICE_NAME_SCHEMA: Schema = StringSchema::new("Block device name
             },
         },
         owner: {
-            type: Userid,
+            type: Authid,
             optional: true,
         },
     },
@@ -382,7 +432,7 @@ pub struct GroupListItem {
     pub files: Vec<String>,
     /// The owner of group
     #[serde(skip_serializing_if="Option::is_none")]
-    pub owner: Option<Userid>,
+    pub owner: Option<Authid>,
 }
 
 #[api()]
@@ -440,7 +490,7 @@ pub struct SnapshotVerifyState {
             },
         },
         owner: {
-            type: Userid,
+            type: Authid,
             optional: true,
         },
     },
@@ -465,7 +515,7 @@ pub struct SnapshotListItem {
     pub size: Option<u64>,
     /// The owner of the snapshots group
     #[serde(skip_serializing_if="Option::is_none")]
-    pub owner: Option<Userid>,
+    pub owner: Option<Authid>,
 }
 
 #[api(
@@ -577,6 +627,8 @@ pub struct GarbageCollectionStatus {
     pub pending_chunks: usize,
     /// Number of chunks marked as .bad by verify that have been removed by GC.
     pub removed_bad: usize,
+    /// Number of chunks still marked as .bad after garbage collection.
+    pub still_bad: usize,
 }
 
 impl Default for GarbageCollectionStatus {
@@ -592,6 +644,7 @@ impl Default for GarbageCollectionStatus {
             pending_bytes: 0,
             pending_chunks: 0,
             removed_bad: 0,
+            still_bad: 0,
         }
     }
 }
@@ -609,10 +662,75 @@ pub struct StorageStatus {
     pub avail: u64,
 }
 
+#[api()]
+#[derive(Serialize, Deserialize, Default)]
+/// Backup Type group/snapshot counts.
+pub struct TypeCounts {
+    /// The number of groups of the type.
+    pub groups: u64,
+    /// The number of snapshots of the type.
+    pub snapshots: u64,
+}
+
+#[api(
+    properties: {
+        ct: {
+            type: TypeCounts,
+            optional: true,
+        },
+        host: {
+            type: TypeCounts,
+            optional: true,
+        },
+        vm: {
+            type: TypeCounts,
+            optional: true,
+        },
+        other: {
+            type: TypeCounts,
+            optional: true,
+        },
+    },
+)]
+#[derive(Serialize, Deserialize)]
+/// Counts of groups/snapshots per BackupType.
+pub struct Counts {
+    /// The counts for CT backups
+    pub ct: Option<TypeCounts>,
+    /// The counts for Host backups
+    pub host: Option<TypeCounts>,
+    /// The counts for VM backups
+    pub vm: Option<TypeCounts>,
+    /// The counts for other backup types
+    pub other: Option<TypeCounts>,
+}
+
+#[api(
+    properties: {
+        "gc-status": { type: GarbageCollectionStatus, },
+        counts: { type: Counts, }
+    },
+)]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all="kebab-case")]
+/// Overall Datastore status and useful information.
+pub struct DataStoreStatus {
+    /// Total space (bytes).
+    pub total: u64,
+    /// Used space (bytes).
+    pub used: u64,
+    /// Available space (bytes).
+    pub avail: u64,
+    /// Status of last GC
+    pub gc_status: GarbageCollectionStatus,
+    /// Group/Snapshot counts
+    pub counts: Counts,
+}
+
 #[api(
     properties: {
         upid: { schema: UPID_SCHEMA },
-        user: { type: Userid },
+        user: { type: Authid },
     },
 )]
 #[derive(Serialize, Deserialize)]
@@ -631,8 +749,8 @@ pub struct TaskListItem {
     pub worker_type: String,
     /// Worker ID (arbitrary ASCII string)
     pub worker_id: Option<String>,
-    /// The user who started the task
-    pub user: Userid,
+    /// The authenticated entity who started the task
+    pub user: Authid,
     /// The task end time (Epoch)
     #[serde(skip_serializing_if="Option::is_none")]
     pub endtime: Option<i64>,
@@ -655,7 +773,7 @@ impl From<crate::server::TaskListInfo> for TaskListItem {
             starttime: info.upid.starttime,
             worker_type: info.upid.worker_type,
             worker_id: info.upid.worker_id,
-            user: info.upid.userid,
+            user: info.upid.auth_id,
             endtime,
             status,
         }
@@ -1035,7 +1153,7 @@ pub enum RRDTimeFrameResolution {
 }
 
 #[api()]
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "PascalCase")]
 /// Describes a package for which an update is available.
 pub struct APTUpdateInfo {
@@ -1060,3 +1178,48 @@ pub struct APTUpdateInfo {
     /// URL under which the package's changelog can be retrieved
     pub change_log_url: String,
 }
+
+#[api()]
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// When do we send notifications
+pub enum Notify {
+    /// Never send notification
+    Never,
+    /// Send notifications for failed and sucessful jobs
+    Always,
+    /// Send notifications for failed jobs only
+    Error,
+}
+
+#[api(
+    properties: {
+        gc: {
+            type: Notify,
+            optional: true,
+        },
+        verify: {
+            type: Notify,
+            optional: true,
+        },
+        sync: {
+            type: Notify,
+            optional: true,
+        },
+    },
+)]
+#[derive(Debug, Serialize, Deserialize)]
+/// Datastore notify settings
+pub struct DatastoreNotify {
+    /// Garbage collection settings
+    pub gc: Option<Notify>,
+    /// Verify job setting
+    pub verify: Option<Notify>,
+    /// Sync job setting
+    pub sync: Option<Notify>,
+}
+
+pub const DATASTORE_NOTIFY_STRING_SCHEMA: Schema = StringSchema::new(
+    "Datastore notification setting")
+    .format(&ApiStringFormat::PropertyString(&DatastoreNotify::API_SCHEMA))
+    .schema();

src/api2/types/userid.rs

@@ -1,6 +1,7 @@
 //! Types for user handling.
 //!
-//! We have [`Username`]s and [`Realm`]s. To uniquely identify a user, they must be combined into a [`Userid`].
+//! We have [`Username`]s, [`Realm`]s and [`Tokenname`]s. To uniquely identify a user/API token, they
+//! must be combined into a [`Userid`] or [`Authid`].
 //!
 //! Since they're all string types, they're organized as follows:
 //!
@@ -9,13 +10,16 @@
 //!   with `String`, meaning you can only make references to it.
 //! * [`Realm`]: an owned realm (`String` equivalent).
 //! * [`RealmRef`]: a borrowed realm (`str` equivalent).
-//! * [`Userid`]: an owned user id (`"user@realm"`). Note that this does not have a separate
-//!   borrowed type.
+//! * [`Tokenname`]: an owned API token name (`String` equivalent)
+//! * [`TokennameRef`]: a borrowed `Tokenname` (`str` equivalent).
+//! * [`Userid`]: an owned user id (`"user@realm"`).
+//! * [`Authid`]: an owned Authentication ID (a `Userid` with an optional `Tokenname`).
+//! Note that `Userid` and `Authid` do not have a separate borrowed type.
 //!
-//! Note that `Username`s are not unique, therefore they do not implement `Eq` and cannot be
+//! Note that `Username`s and `Tokenname`s are not unique, therefore they do not implement `Eq` and cannot be
 //! compared directly. If a direct comparison is really required, they can be compared as strings
-//! via the `as_str()` method. [`Realm`]s and [`Userid`]s on the other hand can be compared with
-//! each other, as in those two cases the comparison has meaning.
+//! via the `as_str()` method. [`Realm`]s, [`Userid`]s and [`Authid`]s on the other
+//! hand can be compared with each other, as in those cases the comparison has meaning.
 
 use std::borrow::Borrow;
 use std::convert::TryFrom;
@@ -36,19 +40,42 @@ use proxmox::const_regex;
 // also see "man useradd"
 macro_rules! USER_NAME_REGEX_STR { () => (r"(?:[^\s:/[:cntrl:]]+)") }
 macro_rules! GROUP_NAME_REGEX_STR { () => (USER_NAME_REGEX_STR!()) }
+macro_rules! TOKEN_NAME_REGEX_STR { () => (PROXMOX_SAFE_ID_REGEX_STR!()) }
 macro_rules! USER_ID_REGEX_STR { () => (concat!(USER_NAME_REGEX_STR!(), r"@", PROXMOX_SAFE_ID_REGEX_STR!())) }
+macro_rules! APITOKEN_ID_REGEX_STR { () => (concat!(USER_ID_REGEX_STR!() , r"!", TOKEN_NAME_REGEX_STR!())) }
 
 const_regex! {
     pub PROXMOX_USER_NAME_REGEX = concat!(r"^",  USER_NAME_REGEX_STR!(), r"$");
+    pub PROXMOX_TOKEN_NAME_REGEX = concat!(r"^", TOKEN_NAME_REGEX_STR!(), r"$");
     pub PROXMOX_USER_ID_REGEX = concat!(r"^",  USER_ID_REGEX_STR!(), r"$");
+    pub PROXMOX_APITOKEN_ID_REGEX = concat!(r"^", APITOKEN_ID_REGEX_STR!(), r"$");
+    pub PROXMOX_AUTH_ID_REGEX = concat!(r"^", r"(?:", USER_ID_REGEX_STR!(), r"|", APITOKEN_ID_REGEX_STR!(), r")$");
     pub PROXMOX_GROUP_ID_REGEX = concat!(r"^",  GROUP_NAME_REGEX_STR!(), r"$");
 }
 
 pub const PROXMOX_USER_NAME_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&PROXMOX_USER_NAME_REGEX);
+pub const PROXMOX_TOKEN_NAME_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&PROXMOX_TOKEN_NAME_REGEX);
 
 pub const PROXMOX_USER_ID_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&PROXMOX_USER_ID_REGEX);
+pub const PROXMOX_TOKEN_ID_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&PROXMOX_APITOKEN_ID_REGEX);
+pub const PROXMOX_AUTH_ID_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&PROXMOX_AUTH_ID_REGEX);
+
+pub const PROXMOX_TOKEN_ID_SCHEMA: Schema = StringSchema::new("API Token ID")
+    .format(&PROXMOX_TOKEN_ID_FORMAT)
+    .min_length(3)
+    .max_length(64)
+    .schema();
+
+pub const PROXMOX_TOKEN_NAME_SCHEMA: Schema = StringSchema::new("API Token name")
+    .format(&PROXMOX_TOKEN_NAME_FORMAT)
+    .min_length(3)
+    .max_length(64)
+    .schema();
 
 pub const PROXMOX_GROUP_ID_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&PROXMOX_GROUP_ID_REGEX);
@@ -91,26 +118,6 @@ pub struct Username(String);
 #[derive(Debug, Hash)]
 pub struct UsernameRef(str);
 
-#[doc(hidden)]
-/// ```compile_fail
-/// let a: Username = unsafe { std::mem::zeroed() };
-/// let b: Username = unsafe { std::mem::zeroed() };
-/// let _ = <Username as PartialEq>::eq(&a, &b);
-/// ```
-///
-/// ```compile_fail
-/// let a: &UsernameRef = unsafe { std::mem::zeroed() };
-/// let b: &UsernameRef = unsafe { std::mem::zeroed() };
-/// let _ = <&UsernameRef as PartialEq>::eq(a, b);
-/// ```
-///
-/// ```compile_fail
-/// let a: &UsernameRef = unsafe { std::mem::zeroed() };
-/// let b: &UsernameRef = unsafe { std::mem::zeroed() };
-/// let _ = <&UsernameRef as PartialEq>::eq(&a, &b);
-/// ```
-struct _AssertNoEqImpl;
-
 impl UsernameRef {
     fn new(s: &str) -> &Self {
         unsafe { &*(s as *const str as *const UsernameRef) }
@@ -286,7 +293,132 @@ impl PartialEq<Realm> for &RealmRef {
     }
 }
 
-/// A complete user id consting of a user name and a realm.
+#[api(
+    type: String,
+    format: &PROXMOX_TOKEN_NAME_FORMAT,
+)]
+/// The token ID part of an API token authentication id.
+///
+/// This alone does NOT uniquely identify the API token and therefore does not implement `Eq`. In
+/// order to compare token IDs directly, they need to be explicitly compared as strings by calling
+/// `.as_str()`.
+///
+/// ```compile_fail
+/// fn test(a: Tokenname, b: Tokenname) -> bool {
+///     a == b // illegal and does not compile
+/// }
+/// ```
+#[derive(Clone, Debug, Hash, Deserialize, Serialize)]
+pub struct Tokenname(String);
+
+/// A reference to a token name part of an authentication id. This alone does NOT uniquely identify
+/// the user.
+///
+/// This is like a `str` to the `String` of a [`Tokenname`].
+#[derive(Debug, Hash)]
+pub struct TokennameRef(str);
+
+#[doc(hidden)]
+/// ```compile_fail
+/// let a: Username = unsafe { std::mem::zeroed() };
+/// let b: Username = unsafe { std::mem::zeroed() };
+/// let _ = <Username as PartialEq>::eq(&a, &b);
+/// ```
+///
+/// ```compile_fail
+/// let a: &UsernameRef = unsafe { std::mem::zeroed() };
+/// let b: &UsernameRef = unsafe { std::mem::zeroed() };
+/// let _ = <&UsernameRef as PartialEq>::eq(a, b);
+/// ```
+///
+/// ```compile_fail
+/// let a: &UsernameRef = unsafe { std::mem::zeroed() };
+/// let b: &UsernameRef = unsafe { std::mem::zeroed() };
+/// let _ = <&UsernameRef as PartialEq>::eq(&a, &b);
+/// ```
+///
+/// ```compile_fail
+/// let a: Tokenname = unsafe { std::mem::zeroed() };
+/// let b: Tokenname = unsafe { std::mem::zeroed() };
+/// let _ = <Tokenname as PartialEq>::eq(&a, &b);
+/// ```
+///
+/// ```compile_fail
+/// let a: &TokennameRef = unsafe { std::mem::zeroed() };
+/// let b: &TokennameRef = unsafe { std::mem::zeroed() };
+/// let _ = <&TokennameRef as PartialEq>::eq(a, b);
+/// ```
+///
+/// ```compile_fail
+/// let a: &TokennameRef = unsafe { std::mem::zeroed() };
+/// let b: &TokennameRef = unsafe { std::mem::zeroed() };
+/// let _ = <&TokennameRef as PartialEq>::eq(&a, &b);
+/// ```
+struct _AssertNoEqImpl;
+
+impl TokennameRef {
+    fn new(s: &str) -> &Self {
+        unsafe { &*(s as *const str as *const TokennameRef) }
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl std::ops::Deref for Tokenname {
+    type Target = TokennameRef;
+
+    fn deref(&self) -> &TokennameRef {
+        self.borrow()
+    }
+}
+
+impl Borrow<TokennameRef> for Tokenname {
+    fn borrow(&self) -> &TokennameRef {
+        TokennameRef::new(self.0.as_str())
+    }
+}
+
+impl AsRef<TokennameRef> for Tokenname {
+    fn as_ref(&self) -> &TokennameRef {
+        self.borrow()
+    }
+}
+
+impl ToOwned for TokennameRef {
+    type Owned = Tokenname;
+
+    fn to_owned(&self) -> Self::Owned {
+        Tokenname(self.0.to_owned())
+    }
+}
+
+impl TryFrom<String> for Tokenname {
+    type Error = Error;
+
+    fn try_from(s: String) -> Result<Self, Error> {
+        if !PROXMOX_TOKEN_NAME_REGEX.is_match(&s) {
+            bail!("invalid token name");
+        }
+
+        Ok(Self(s))
+    }
+}
+
+impl<'a> TryFrom<&'a str> for &'a TokennameRef {
+    type Error = Error;
+
+    fn try_from(s: &'a str) -> Result<&'a TokennameRef, Error> {
+        if !PROXMOX_TOKEN_NAME_REGEX.is_match(s) {
+            bail!("invalid token name in user id");
+        }
+
+        Ok(TokennameRef::new(s))
+    }
+}
+
+/// A complete user id consisting of a user name and a realm
 #[derive(Clone, Debug, Hash)]
 pub struct Userid {
     data: String,
@@ -318,11 +450,6 @@ impl Userid {
         &self.data
     }
 
-    /// Get the "backup@pam" user id.
-    pub fn backup_userid() -> &'static Self {
-        &*BACKUP_USERID
-    }
-
     /// Get the "root@pam" user id.
     pub fn root_userid() -> &'static Self {
         &*ROOT_USERID
@@ -330,7 +457,6 @@ impl Userid {
 }
 
 lazy_static! {
-    pub static ref BACKUP_USERID: Userid = Userid::new("backup@pam".to_string(), 6);
     pub static ref ROOT_USERID: Userid = Userid::new("root@pam".to_string(), 4);
 }
 
@@ -342,6 +468,12 @@ impl PartialEq for Userid {
     }
 }
 
+impl From<Authid> for Userid {
+    fn from(authid: Authid) -> Self {
+        authid.user
+    }
+}
+
 impl From<(Username, Realm)> for Userid {
     fn from(parts: (Username, Realm)) -> Self {
         Self::from((parts.0.as_ref(), parts.1.as_ref()))
@@ -366,10 +498,18 @@ impl std::str::FromStr for Userid {
     type Err = Error;
 
     fn from_str(id: &str) -> Result<Self, Error> {
-        let (name, realm) = match id.as_bytes().iter().rposition(|&b| b == b'@') {
-            Some(pos) => (&id[..pos], &id[(pos + 1)..]),
-            None => bail!("not a valid user id"),
-        };
+        let name_len = id
+            .as_bytes()
+            .iter()
+            .rposition(|&b| b == b'@')
+            .ok_or_else(|| format_err!("not a valid user id"))?;
+
+        let name = &id[..name_len];
+        let realm = &id[(name_len + 1)..];
+
+        if !PROXMOX_USER_NAME_REGEX.is_match(name) {
+            bail!("invalid user name in user id");
+        }
 
         PROXMOX_AUTH_REALM_STRING_SCHEMA.check_constraints(realm)
             .map_err(|_| format_err!("invalid realm in user id"))?;
@@ -388,6 +528,10 @@ impl TryFrom<String> for Userid {
             .rposition(|&b| b == b'@')
             .ok_or_else(|| format_err!("not a valid user id"))?;
 
+        if !PROXMOX_USER_NAME_REGEX.is_match(&data[..name_len]) {
+            bail!("invalid user name in user id");
+        }
+
         PROXMOX_AUTH_REALM_STRING_SCHEMA.check_constraints(&data[(name_len + 1)..])
             .map_err(|_| format_err!("invalid realm in user id"))?;
 
@@ -413,5 +557,176 @@ impl PartialEq<String> for Userid {
     }
 }
 
+/// A complete authentication id consisting of a user id and an optional token name.
+#[derive(Clone, Debug, Hash)]
+pub struct Authid {
+    user: Userid,
+    tokenname: Option<Tokenname>
+}
+
+impl Authid {
+    pub const API_SCHEMA: Schema = StringSchema::new("Authentication ID")
+        .format(&PROXMOX_AUTH_ID_FORMAT)
+        .min_length(3)
+        .max_length(64)
+        .schema();
+
+    const fn new(user: Userid, tokenname: Option<Tokenname>) -> Self {
+        Self { user, tokenname }
+    }
+
+    pub fn user(&self) -> &Userid {
+        &self.user
+    }
+
+    pub fn is_token(&self) -> bool {
+        self.tokenname.is_some()
+    }
+
+    pub fn tokenname(&self) -> Option<&TokennameRef> {
+        match &self.tokenname {
+            Some(name) => Some(&name),
+            None => None,
+        }
+    }
+
+    /// Get the "root@pam" auth id.
+    pub fn root_auth_id() -> &'static Self {
+        &*ROOT_AUTHID
+    }
+}
+
+lazy_static! {
+    pub static ref ROOT_AUTHID: Authid = Authid::from(Userid::new("root@pam".to_string(), 4));
+}
+
+impl Eq for Authid {}
+
+impl PartialEq for Authid {
+    fn eq(&self, rhs: &Self) -> bool {
+        self.user == rhs.user && match (&self.tokenname, &rhs.tokenname) {
+            (Some(ours), Some(theirs)) => ours.as_str() == theirs.as_str(),
+            (None, None) => true,
+            _ => false,
+        }
+    }
+}
+
+impl From<Userid> for Authid {
+    fn from(parts: Userid) -> Self {
+        Self::new(parts, None)
+    }
+}
+
+impl From<(Userid, Option<Tokenname>)> for Authid {
+    fn from(parts: (Userid, Option<Tokenname>)) -> Self {
+        Self::new(parts.0, parts.1)
+    }
+}
+
+impl fmt::Display for Authid {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.tokenname {
+            Some(token) => write!(f, "{}!{}", self.user, token.as_str()),
+            None => self.user.fmt(f),
+        }
+    }
+}
+
+impl std::str::FromStr for Authid {
+    type Err = Error;
+
+    fn from_str(id: &str) -> Result<Self, Error> {
+        let name_len = id
+            .as_bytes()
+            .iter()
+            .rposition(|&b| b == b'@')
+            .ok_or_else(|| format_err!("not a valid user id"))?;
+
+        let realm_end = id
+            .as_bytes()
+            .iter()
+            .rposition(|&b| b == b'!')
+            .map(|pos| if pos < name_len { id.len() } else { pos })
+            .unwrap_or(id.len());
+
+        if realm_end == id.len() - 1 {
+            bail!("empty token name in userid");
+        }
+
+        let user = Userid::from_str(&id[..realm_end])?;
+
+        if id.len() > realm_end {
+            let token = Tokenname::try_from(id[(realm_end + 1)..].to_string())?;
+            Ok(Self::new(user, Some(token)))
+        } else {
+            Ok(Self::new(user, None))
+        }
+    }
+}
+
+impl TryFrom<String> for Authid {
+    type Error = Error;
+
+    fn try_from(mut data: String) -> Result<Self, Error> {
+        let name_len = data
+            .as_bytes()
+            .iter()
+            .rposition(|&b| b == b'@')
+            .ok_or_else(|| format_err!("not a valid user id"))?;
+
+        let realm_end = data
+            .as_bytes()
+            .iter()
+            .rposition(|&b| b == b'!')
+            .map(|pos| if pos < name_len { data.len() } else { pos })
+            .unwrap_or(data.len());
+
+        if realm_end == data.len() - 1 {
+            bail!("empty token name in userid");
+        }
+
+        let tokenname = if data.len() > realm_end {
+            Some(Tokenname::try_from(data[(realm_end + 1)..].to_string())?)
+        } else {
+            None
+        };
+
+        data.truncate(realm_end);
+
+        let user:Userid = data.parse()?;
+
+        Ok(Self { user, tokenname })
+    }
+}
+
+#[test]
+fn test_token_id() {
+    let userid: Userid = "test@pam".parse().expect("parsing Userid failed");
+    assert_eq!(userid.name().as_str(), "test");
+    assert_eq!(userid.realm(), "pam");
+    assert_eq!(userid, "test@pam");
+
+    let auth_id: Authid = "test@pam".parse().expect("parsing user Authid failed");
+    assert_eq!(auth_id.to_string(), "test@pam".to_string());
+    assert!(!auth_id.is_token());
+
+    assert_eq!(auth_id.user(), &userid);
+
+    let user_auth_id = Authid::from(userid.clone());
+    assert_eq!(user_auth_id, auth_id);
+    assert!(!user_auth_id.is_token());
+
+    let auth_id: Authid = "test@pam!bar".parse().expect("parsing token Authid failed");
+    let token_userid = auth_id.user();
+    assert_eq!(&userid, token_userid);
+    assert!(auth_id.is_token());
+    assert_eq!(auth_id.tokenname().expect("Token has tokenname").as_str(), TokennameRef::new("bar").as_str());
+    assert_eq!(auth_id.to_string(), "test@pam!bar".to_string());
+}
+
 proxmox::forward_deserialize_to_from_str!(Userid);
 proxmox::forward_serialize_to_display!(Userid);
+
+proxmox::forward_deserialize_to_from_str!(Authid);
+proxmox::forward_serialize_to_display!(Authid);

src/backup.rs

@@ -1,107 +1,146 @@
-//! This module implements the proxmox backup data storage
+//! This module implements the data storage and access layer.
 //!
-//! Proxmox backup splits large files into chunks, and stores them
-//! deduplicated using a content addressable storage format.
+//! # Data formats
 //!
-//! A chunk is simply defined as binary blob, which is stored inside a
-//! `ChunkStore`, addressed by the SHA256 digest of the binary blob.
+//! PBS splits large files into chunks, and stores them deduplicated using
+//! a content addressable storage format.
 //!
-//! Index files are used to reconstruct the original file. They
-//! basically contain a list of SHA256 checksums. The `DynamicIndex*`
-//! format is able to deal with dynamic chunk sizes, whereas the
-//! `FixedIndex*` format is an optimization to store a list of equal
-//! sized chunks.
+//! Backup snapshots are stored as folders containing a manifest file and
+//! potentially one or more index or blob files.
 //!
-//! # ChunkStore Locking
+//! The manifest contains hashes of all other files and can be signed by
+//! the client.
 //!
-//! We need to be able to restart the proxmox-backup service daemons,
-//! so that we can update the software without rebooting the host. But
-//! such restarts must not abort running backup jobs, so we need to
-//! keep the old service running until those jobs are finished. This
-//! implies that we need some kind of locking for the
-//! ChunkStore. Please note that it is perfectly valid to have
-//! multiple parallel ChunkStore writers, even when they write the
-//! same chunk (because the chunk would have the same name and the
-//! same data). The only real problem is garbage collection, because
-//! we need to avoid deleting chunks which are still referenced.
+//! Blob files contain data directly. They are used for config files and
+//! the like.
 //!
-//! * Read Index Files:
+//! Index files are used to reconstruct an original file. They contain a
+//! list of SHA256 checksums. The `DynamicIndex*` format is able to deal
+//! with dynamic chunk sizes (CT and host backups), whereas the
+//! `FixedIndex*` format is an optimization to store a list of equal sized
+//! chunks (VMs, whole block devices).
 //!
-//!   Acquire shared lock for .idx files.
-//!
-//!
-//! * Delete Index Files:
-//!
-//!   Acquire exclusive lock for .idx files. This makes sure that we do
-//!   not delete index files while they are still in use.
-//!
-//!
-//! * Create Index Files:
-//!
-//!   Acquire shared lock for ChunkStore (process wide).
-//!
-//!   Note: When creating .idx files, we create temporary a (.tmp) file,
-//!   then do an atomic rename ...
-//!
-//!
-//! * Garbage Collect:
-//!
-//!   Acquire exclusive lock for ChunkStore (process wide). If we have
-//!   already a shared lock for the ChunkStore, try to upgrade that
-//!   lock.
-//!
-//!
-//! * Server Restart
-//!
-//!   Try to abort the running garbage collection to release exclusive
-//!   ChunkStore locks ASAP. Start the new service with the existing listening
-//!   socket.
+//! A chunk is defined as a binary blob, which is stored inside a
+//! [ChunkStore](struct.ChunkStore.html) instead of the backup directory
+//! directly, and can be addressed by its SHA256 digest.
 //!
 //!
 //! # Garbage Collection (GC)
 //!
-//! Deleting backups is as easy as deleting the corresponding .idx
-//! files. Unfortunately, this does not free up any storage, because
-//! those files just contain references to chunks.
+//! Deleting backups is as easy as deleting the corresponding .idx files.
+//! However, this does not free up any storage, because those files just
+//! contain references to chunks.
 //!
 //! To free up some storage, we run a garbage collection process at
-//! regular intervals. The collector uses a mark and sweep
-//! approach. In the first phase, it scans all .idx files to mark used
-//! chunks. The second phase then removes all unmarked chunks from the
-//! store.
+//! regular intervals. The collector uses a mark and sweep approach. In
+//! the first phase, it scans all .idx files to mark used chunks. The
+//! second phase then removes all unmarked chunks from the store.
 //!
-//! The above locking mechanism makes sure that we are the only
-//! process running GC. But we still want to be able to create backups
-//! during GC, so there may be multiple backup threads/tasks
-//! running. Either started before GC started, or started while GC is
-//! running.
+//! The locking mechanisms mentioned below make sure that we are the only
+//! process running GC. We still want to be able to create backups during
+//! GC, so there may be multiple backup threads/tasks running, either
+//! started before GC, or while GC is running.
 //!
 //! ## `atime` based GC
 //!
 //! The idea here is to mark chunks by updating the `atime` (access
-//! timestamp) on the chunk file. This is quite simple and does not
-//! need additional RAM.
+//! timestamp) on the chunk file. This is quite simple and does not need
+//! additional RAM.
 //!
 //! One minor problem is that recent Linux versions use the `relatime`
-//! mount flag by default for performance reasons (yes, we want
-//! that). When enabled, `atime` data is written to the disk only if
-//! the file has been modified since the `atime` data was last updated
-//! (`mtime`), or if the file was last accessed more than a certain
-//! amount of time ago (by default 24h). So we may only delete chunks
-//! with `atime` older than 24 hours.
-//!
-//! Another problem arises from running backups. The mark phase does
-//! not find any chunks from those backups, because there is no .idx
-//! file for them (created after the backup). Chunks created or
-//! touched by those backups may have an `atime` as old as the start
-//! time of those backups. Please note that the backup start time may
-//! predate the GC start time. So we may only delete chunks older than
-//! the start time of those running backup jobs.
+//! mount flag by default for performance reasons (and we want that). When
+//! enabled, `atime` data is written to the disk only if the file has been
+//! modified since the `atime` data was last updated (`mtime`), or if the
+//! file was last accessed more than a certain amount of time ago (by
+//! default 24h). So we may only delete chunks with `atime` older than 24
+//! hours.
 //!
+//! Another problem arises from running backups. The mark phase does not
+//! find any chunks from those backups, because there is no .idx file for
+//! them (created after the backup). Chunks created or touched by those
+//! backups may have an `atime` as old as the start time of those backups.
+//! Please note that the backup start time may predate the GC start time.
+//! So we may only delete chunks older than the start time of those
+//! running backup jobs, which might be more than 24h back (this is the
+//! reason why ProcessLocker exclusive locks only have to be exclusive
+//! between processes, since within one we can determine the age of the
+//! oldest shared lock).
 //!
 //! ## Store `marks` in RAM using a HASH
 //!
-//! Not sure if this is better. TODO
+//! Might be better. Under investigation.
+//!
+//!
+//! # Locking
+//!
+//! Since PBS allows multiple potentially interfering operations at the
+//! same time (e.g. garbage collect, prune, multiple backup creations
+//! (only in seperate groups), forget, ...), these need to lock against
+//! each other in certain scenarios. There is no overarching global lock
+//! though, instead always the finest grained lock possible is used,
+//! because running these operations concurrently is treated as a feature
+//! on its own.
+//!
+//! ## Inter-process Locking
+//!
+//! We need to be able to restart the proxmox-backup service daemons, so
+//! that we can update the software without rebooting the host. But such
+//! restarts must not abort running backup jobs, so we need to keep the
+//! old service running until those jobs are finished. This implies that
+//! we need some kind of locking for modifying chunks and indices in the
+//! ChunkStore.
+//!
+//! Please note that it is perfectly valid to have multiple
+//! parallel ChunkStore writers, even when they write the same chunk
+//! (because the chunk would have the same name and the same data, and
+//! writes are completed atomically via a rename). The only problem is
+//! garbage collection, because we need to avoid deleting chunks which are
+//! still referenced.
+//!
+//! To do this we use the
+//! [ProcessLocker](../tools/struct.ProcessLocker.html).
+//!
+//! ### ChunkStore-wide
+//!
+//! * Create Index Files:
+//!
+//!   Acquire shared lock for ChunkStore.
+//!
+//!   Note: When creating .idx files, we create a temporary .tmp file,
+//!   then do an atomic rename.
+//!
+//! * Garbage Collect:
+//!
+//!   Acquire exclusive lock for ChunkStore. If we have
+//!   already a shared lock for the ChunkStore, try to upgrade that
+//!   lock.
+//!
+//! Exclusive locks only work _between processes_. It is valid to have an
+//! exclusive and one or more shared locks held within one process. Writing
+//! chunks within one process is synchronized using the gc_mutex.
+//!
+//! On server restart, we stop any running GC in the old process to avoid
+//! having the exclusive lock held for too long.
+//!
+//! ## Locking table
+//!
+//! Below table shows all operations that play a role in locking, and which
+//! mechanisms are used to make their concurrent usage safe.
+//!
+//! | starting ><br>v during | read index file | create index file | GC mark | GC sweep | update manifest | forget | prune | create backup | verify | reader api |
+//! |-|-|-|-|-|-|-|-|-|-|-|
+//! | **read index file** | / | / | / | / | / | mmap stays valid, oldest_shared_lock prevents GC | see forget column | / | / | / |
+//! | **create index file** | / | / | / | / | / | / | / | /, happens at the end, after all chunks are touched | /, only happens without a manifest | / |
+//! | **GC mark** | / | Datastore process-lock shared | gc_mutex, exclusive ProcessLocker | gc_mutex | /, GC only cares about index files, not manifests | tells GC about removed chunks | see forget column | /, index files don’t exist yet | / | / |
+//! | **GC sweep** | / | Datastore process-lock shared | gc_mutex, exclusive ProcessLocker | gc_mutex | / | /, chunks already marked | see forget column | chunks get touched; chunk_store.mutex; oldest PL lock | / | / |
+//! | **update manifest** | / | / | / | / | update_manifest lock | update_manifest lock, remove dir under lock | see forget column | /, “write manifest” happens at the end | /, can call “write manifest”, see that column | / |
+//! | **forget** | / | / | removed_during_gc mutex is held during unlink | marking done, doesn’t matter if forgotten now | update_manifest lock, forget waits for lock | /, unlink is atomic | causes forget to fail, but that’s OK | running backup has snapshot flock | /, potentially detects missing folder | shared snap flock |
+//! | **prune** | / | / | see forget row | see forget row | see forget row | causes warn in prune, but no error | see forget column | running and last non-running can’t be pruned | see forget row | shared snap flock |
+//! | **create backup** | / | only time this happens, thus has snapshot flock | / | chunks get touched; chunk_store.mutex; oldest PL lock | no lock, but cannot exist beforehand | snapshot flock, can’t be forgotten | running and last non-running can’t be pruned | snapshot group flock, only one running per group | /, won’t be verified since manifest missing | / |
+//! | **verify** | / | / | / | / | see “update manifest” row | /, potentially detects missing folder | see forget column | / | /, but useless (“update manifest” protects itself) | / |
+//! | **reader api** | / | / | / | /, open snap can’t be forgotten, so ref must exist | / | prevented by shared snap flock | prevented by shared snap flock | / | / | /, lock is shared |!
+//! * / = no interaction
+//! * shared/exclusive from POV of 'starting' process
 
 use anyhow::{bail, Error};
 

src/backup/async_index_reader.rs

@@ -15,6 +15,17 @@ use super::IndexFile;
 use super::read_chunk::AsyncReadChunk;
 use super::index::ChunkReadInfo;
 
+// FIXME: This enum may not be required?
+// - Put the `WaitForData` case directly into a `read_future: Option<>`
+// - make the read loop as follows:
+//   * if read_buffer is not empty:
+//        use it
+//   * else if read_future is there:
+//        poll it
+//        if read: move data to read_buffer
+//   * else
+//        create read future
+#[allow(clippy::enum_variant_names)]
 enum AsyncIndexReaderState<S> {
     NoData,
     WaitForData(Pin<Box<dyn Future<Output = Result<(S, Vec<u8>), Error>> + Send + 'static>>),
@@ -118,9 +129,8 @@ where
                 }
                 AsyncIndexReaderState::WaitForData(ref mut future) => {
                     match ready!(future.as_mut().poll(cx)) {
-                        Ok((store, mut chunk_data)) => {
-                            this.read_buffer.clear();
-                            this.read_buffer.append(&mut chunk_data);
+                        Ok((store, chunk_data)) => {
+                            this.read_buffer = chunk_data;
                             this.state = AsyncIndexReaderState::HaveData;
                             this.store = Some(store);
                         }

src/backup/backup_info.rs

@@ -1,37 +1,31 @@
 use crate::tools;
 
 use anyhow::{bail, format_err, Error};
-use regex::Regex;
 use std::os::unix::io::RawFd;
 
 use std::path::{PathBuf, Path};
-use lazy_static::lazy_static;
+
+use proxmox::const_regex;
 
 use super::manifest::MANIFEST_BLOB_NAME;
 
-macro_rules! BACKUP_ID_RE { () => (r"[A-Za-z0-9][A-Za-z0-9_-]+") }
+macro_rules! BACKUP_ID_RE { () => (r"[A-Za-z0-9_][A-Za-z0-9._\-]*") }
 macro_rules! BACKUP_TYPE_RE { () => (r"(?:host|vm|ct)") }
 macro_rules! BACKUP_TIME_RE { () => (r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z") }
 
-lazy_static!{
-    static ref BACKUP_FILE_REGEX: Regex = Regex::new(
-        r"^.*\.([fd]idx|blob)$").unwrap();
+const_regex!{
+    BACKUP_FILE_REGEX = r"^.*\.([fd]idx|blob)$";
 
-    static ref BACKUP_TYPE_REGEX: Regex = Regex::new(
-        concat!(r"^(", BACKUP_TYPE_RE!(), r")$")).unwrap();
+    BACKUP_TYPE_REGEX = concat!(r"^(", BACKUP_TYPE_RE!(), r")$");
 
-    static ref BACKUP_ID_REGEX: Regex = Regex::new(
-        concat!(r"^", BACKUP_ID_RE!(), r"$")).unwrap();
+    pub BACKUP_ID_REGEX = concat!(r"^", BACKUP_ID_RE!(), r"$");
 
-    static ref BACKUP_DATE_REGEX: Regex = Regex::new(
-        concat!(r"^", BACKUP_TIME_RE!() ,r"$")).unwrap();
+    BACKUP_DATE_REGEX = concat!(r"^", BACKUP_TIME_RE!() ,r"$");
 
-    static ref GROUP_PATH_REGEX: Regex = Regex::new(
-        concat!(r"^(", BACKUP_TYPE_RE!(), ")/(", BACKUP_ID_RE!(), r")$")).unwrap();
-
-    static ref SNAPSHOT_PATH_REGEX: Regex = Regex::new(
-        concat!(r"^(", BACKUP_TYPE_RE!(), ")/(", BACKUP_ID_RE!(), ")/(", BACKUP_TIME_RE!(), r")$")).unwrap();
+    GROUP_PATH_REGEX = concat!(r"^(", BACKUP_TYPE_RE!(), ")/(", BACKUP_ID_RE!(), r")$");
 
+    SNAPSHOT_PATH_REGEX = concat!(
+        r"^(", BACKUP_TYPE_RE!(), ")/(", BACKUP_ID_RE!(), ")/(", BACKUP_TIME_RE!(), r")$");
 }
 
 /// BackupGroup is a directory containing a list of BackupDir

src/backup/catalog.rs

@@ -78,7 +78,7 @@ pub struct DirEntry {
 #[derive(Clone, Debug, PartialEq)]
 pub enum DirEntryAttribute {
     Directory { start: u64 },
-    File { size: u64, mtime: u64 },
+    File { size: u64, mtime: i64 },
     Symlink,
     Hardlink,
     BlockDevice,
@@ -89,7 +89,7 @@ pub enum DirEntryAttribute {
 
 impl DirEntry {
 
-    fn new(etype: CatalogEntryType, name: Vec<u8>, start: u64, size: u64, mtime:u64) -> Self {
+    fn new(etype: CatalogEntryType, name: Vec<u8>, start: u64, size: u64, mtime: i64) -> Self {
         match etype {
             CatalogEntryType::Directory => {
                 DirEntry { name, attr: DirEntryAttribute::Directory { start } }
@@ -184,7 +184,7 @@ impl DirInfo {
                 catalog_encode_u64(writer, name.len() as u64)?;
                 writer.write_all(name)?;
                 catalog_encode_u64(writer, *size)?;
-                catalog_encode_u64(writer, *mtime)?;
+                catalog_encode_i64(writer, *mtime)?;
             }
             DirEntry { name, attr: DirEntryAttribute::Symlink } => {
                 writer.write_all(&[CatalogEntryType::Symlink as u8])?;
@@ -234,7 +234,7 @@ impl DirInfo {
         Ok((self.name, data))
     }
 
-    fn parse<C: FnMut(CatalogEntryType, &[u8], u64, u64, u64) -> Result<bool, Error>>(
+    fn parse<C: FnMut(CatalogEntryType, &[u8], u64, u64, i64) -> Result<bool, Error>>(
         data: &[u8],
         mut callback: C,
     ) -> Result<(), Error> {
@@ -265,7 +265,7 @@ impl DirInfo {
                 }
                 CatalogEntryType::File => {
                     let size = catalog_decode_u64(&mut cursor)?;
-                    let mtime = catalog_decode_u64(&mut cursor)?;
+                    let mtime = catalog_decode_i64(&mut cursor)?;
                     callback(etype, name, 0, size, mtime)?
                 }
                 _ => {
@@ -362,7 +362,7 @@ impl <W: Write> BackupCatalogWriter for CatalogWriter<W> {
         Ok(())
     }
 
-    fn add_file(&mut self, name: &CStr, size: u64, mtime: u64) -> Result<(), Error> {
+    fn add_file(&mut self, name: &CStr, size: u64, mtime: i64) -> Result<(), Error> {
         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
         let name = name.to_bytes().to_vec();
         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::File { size, mtime } });
@@ -587,14 +587,77 @@ impl <R: Read + Seek> CatalogReader<R> {
     }
 }
 
+/// Serialize i64 as short, variable length byte sequence
+///
+/// Stores 7 bits per byte, Bit 8 indicates the end of the sequence (when not set).
+/// If the value is negative, we end with a zero byte (0x00).
+pub fn catalog_encode_i64<W: Write>(writer: &mut W, v: i64) -> Result<(), Error> {
+    let mut enc = Vec::new();
+
+    let mut d = if v < 0 {
+        (-1 * (v + 1)) as u64 + 1 // also handles i64::MIN
+    } else {
+        v as u64
+    };
+
+    loop {
+        if d < 128 {
+            if v < 0 {
+                enc.push(128 | d as u8);
+                enc.push(0u8);
+            } else {
+                enc.push(d as u8);
+            }
+            break;
+        }
+        enc.push((128 | (d & 127)) as u8);
+        d = d >> 7;
+    }
+    writer.write_all(&enc)?;
+
+    Ok(())
+}
+
+/// Deserialize i64 from variable length byte sequence
+///
+/// We currently read maximal 11 bytes, which give a maximum of 70 bits + sign.
+/// this method is compatible with catalog_encode_u64 iff the
+/// value encoded is <= 2^63 (values > 2^63 cannot be represented in an i64)
+pub fn catalog_decode_i64<R: Read>(reader: &mut R) -> Result<i64, Error> {
+
+    let mut v: u64 = 0;
+    let mut buf = [0u8];
+
+    for i in 0..11 { // only allow 11 bytes (70 bits + sign marker)
+        if buf.is_empty() {
+            bail!("decode_i64 failed - unexpected EOB");
+        }
+        reader.read_exact(&mut buf)?;
+
+        let t = buf[0];
+
+        if t == 0 {
+            if v == 0 {
+                return Ok(0);
+            }
+            return Ok(((v - 1) as i64 * -1) - 1); // also handles i64::MIN
+        } else if t < 128 {
+            v |= (t as u64) << (i*7);
+            return Ok(v as i64);
+        } else {
+            v |= ((t & 127) as u64) << (i*7);
+        }
+    }
+
+    bail!("decode_i64 failed - missing end marker");
+}
+
 /// Serialize u64 as short, variable length byte sequence
 ///
 /// Stores 7 bits per byte, Bit 8 indicates the end of the sequence (when not set).
-/// We limit values to a maximum of 2^63.
 pub fn catalog_encode_u64<W: Write>(writer: &mut W, v: u64) -> Result<(), Error> {
     let mut enc = Vec::new();
 
-    if (v & (1<<63)) != 0 { bail!("catalog_encode_u64 failed - value >= 2^63"); }
     let mut d = v;
     loop {
         if d < 128 {
@@ -611,13 +674,14 @@ pub fn catalog_encode_u64<W: Write>(writer: &mut W, v: u64) -> Result<(), Error>
 
 /// Deserialize u64 from variable length byte sequence
 ///
-/// We currently read maximal 9 bytes, which give a maximum of 63 bits.
+/// We currently read maximal 10 bytes, which give a maximum of 70 bits,
+/// but we currently only encode up to 64 bits
 pub fn catalog_decode_u64<R: Read>(reader: &mut R) -> Result<u64, Error> {
 
     let mut v: u64 = 0;
     let mut buf = [0u8];
 
-    for i in 0..9 { // only allow 9 bytes (63 bits)
+    for i in 0..10 { // only allow 10 bytes (70 bits)
         if buf.is_empty() {
             bail!("decode_u64 failed - unexpected EOB");
         }
@@ -652,9 +716,58 @@ fn test_catalog_u64_encoder() {
         assert!(decoded == value);
     }
 
+    test_encode_decode(u64::MIN);
     test_encode_decode(126);
     test_encode_decode((1<<12)-1);
     test_encode_decode((1<<20)-1);
     test_encode_decode((1<<50)-1);
-    test_encode_decode((1<<63)-1);
+    test_encode_decode(u64::MAX);
+}
+
+#[test]
+fn test_catalog_i64_encoder() {
+
+    fn test_encode_decode(value: i64) {
+
+        let mut data = Vec::new();
+        catalog_encode_i64(&mut data, value).unwrap();
+
+        let slice = &mut &data[..];
+        let decoded = catalog_decode_i64(slice).unwrap();
+
+        assert!(decoded == value);
+    }
+
+    test_encode_decode(0);
+    test_encode_decode(-0);
+    test_encode_decode(126);
+    test_encode_decode(-126);
+    test_encode_decode((1<<12)-1);
+    test_encode_decode(-(1<<12)-1);
+    test_encode_decode((1<<20)-1);
+    test_encode_decode(-(1<<20)-1);
+    test_encode_decode(i64::MIN);
+    test_encode_decode(i64::MAX);
+}
+
+#[test]
+fn test_catalog_i64_compatibility() {
+
+    fn test_encode_decode(value: u64) {
+
+        let mut data = Vec::new();
+        catalog_encode_u64(&mut data, value).unwrap();
+
+        let slice = &mut &data[..];
+        let decoded = catalog_decode_i64(slice).unwrap() as u64;
+
+        assert!(decoded == value);
+    }
+
+    test_encode_decode(u64::MIN);
+    test_encode_decode(126);
+    test_encode_decode((1<<12)-1);
+    test_encode_decode((1<<20)-1);
+    test_encode_decode((1<<50)-1);
+    test_encode_decode(u64::MAX);
 }

src/backup/chunk_store.rs

@@ -354,9 +354,11 @@ impl ChunkStore {
                         },
                         Err(nix::Error::Sys(nix::errno::Errno::ENOENT)) => {
                             // chunk hasn't been rewritten yet, keep .bad file
+                            status.still_bad += 1;
                         },
                         Err(err) => {
                             // some other error, warn user and keep .bad file around too
+                            status.still_bad += 1;
                             crate::task_warn!(
                                 worker,
                                 "error during stat on '{:?}' - {}",
@@ -378,14 +380,12 @@ impl ChunkStore {
                     }
                     status.removed_chunks += 1;
                     status.removed_bytes += stat.st_size as u64;
+                } else if stat.st_atime < oldest_writer {
+                    status.pending_chunks += 1;
+                    status.pending_bytes += stat.st_size as u64;
                 } else {
-                    if stat.st_atime < oldest_writer {
-                        status.pending_chunks += 1;
-                        status.pending_bytes += stat.st_size as u64;
-                    } else {
-                        status.disk_chunks += 1;
-                        status.disk_bytes += stat.st_size as u64;
-                     }
+                    status.disk_chunks += 1;
+                    status.disk_bytes += stat.st_size as u64;
                 }
             }
             drop(lock);

src/backup/datastore.rs

@@ -3,26 +3,27 @@ use std::io::{self, Write};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 use std::convert::TryFrom;
+use std::time::Duration;
+use std::fs::File;
 
 use anyhow::{bail, format_err, Error};
 use lazy_static::lazy_static;
-use serde_json::Value;
 
-use proxmox::tools::fs::{replace_file, CreateOptions};
+use proxmox::tools::fs::{replace_file, file_read_optional_string, CreateOptions, open_file_locked};
 
 use super::backup_info::{BackupGroup, BackupDir};
 use super::chunk_store::ChunkStore;
 use super::dynamic_index::{DynamicIndexReader, DynamicIndexWriter};
 use super::fixed_index::{FixedIndexReader, FixedIndexWriter};
-use super::manifest::{MANIFEST_BLOB_NAME, CLIENT_LOG_BLOB_NAME, BackupManifest};
+use super::manifest::{MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME, CLIENT_LOG_BLOB_NAME, BackupManifest};
 use super::index::*;
 use super::{DataBlob, ArchiveType, archive_type};
-use crate::config::datastore;
+use crate::config::datastore::{self, DataStoreConfig};
 use crate::task::TaskState;
 use crate::tools;
 use crate::tools::format::HumanByte;
 use crate::tools::fs::{lock_dir_noblock, DirLockGuard};
-use crate::api2::types::{GarbageCollectionStatus, Userid};
+use crate::api2::types::{Authid, GarbageCollectionStatus};
 use crate::server::UPID;
 
 lazy_static! {
@@ -37,6 +38,7 @@ pub struct DataStore {
     chunk_store: Arc<ChunkStore>,
     gc_mutex: Mutex<bool>,
     last_gc_status: Mutex<GarbageCollectionStatus>,
+    verify_new: bool,
 }
 
 impl DataStore {
@@ -45,17 +47,20 @@ impl DataStore {
 
         let (config, _digest) = datastore::config()?;
         let config: datastore::DataStoreConfig = config.lookup("datastore", name)?;
+        let path = PathBuf::from(&config.path);
 
         let mut map = DATASTORE_MAP.lock().unwrap();
 
         if let Some(datastore) = map.get(name) {
             // Compare Config - if changed, create new Datastore object!
-            if datastore.chunk_store.base == PathBuf::from(&config.path) {
+            if datastore.chunk_store.base == path &&
+                datastore.verify_new == config.verify_new.unwrap_or(false)
+            {
                 return Ok(datastore.clone());
             }
         }
 
-        let datastore = DataStore::open(name)?;
+        let datastore = DataStore::open_with_path(name, &path, config)?;
 
         let datastore = Arc::new(datastore);
         map.insert(name.to_string(), datastore.clone());
@@ -63,26 +68,29 @@ impl DataStore {
         Ok(datastore)
     }
 
-    pub fn open(store_name: &str) -> Result<Self, Error> {
-
-        let (config, _digest) = datastore::config()?;
-        let (_, store_config) = config.sections.get(store_name)
-            .ok_or(format_err!("no such datastore '{}'", store_name))?;
-
-        let path = store_config["path"].as_str().unwrap();
-
-        Self::open_with_path(store_name, Path::new(path))
-    }
-
-    pub fn open_with_path(store_name: &str, path: &Path) -> Result<Self, Error> {
+    fn open_with_path(store_name: &str, path: &Path, config: DataStoreConfig) -> Result<Self, Error> {
         let chunk_store = ChunkStore::open(store_name, path)?;
 
-        let gc_status = GarbageCollectionStatus::default();
+        let mut gc_status_path = chunk_store.base_path();
+        gc_status_path.push(".gc-status");
+
+        let gc_status = if let Some(state) = file_read_optional_string(gc_status_path)? {
+            match serde_json::from_str(&state) {
+                Ok(state) => state,
+                Err(err) => {
+                    eprintln!("error reading gc-status: {}", err);
+                    GarbageCollectionStatus::default()
+                }
+            }
+        } else {
+            GarbageCollectionStatus::default()
+        };
 
         Ok(Self {
             chunk_store: Arc::new(chunk_store),
             gc_mutex: Mutex::new(false),
             last_gc_status: Mutex::new(gc_status),
+            verify_new: config.verify_new.unwrap_or(false),
         })
     }
 
@@ -208,10 +216,17 @@ impl DataStore {
         let _guard = tools::fs::lock_dir_noblock(&full_path, "backup group", "possible running backup")?;
 
         log::info!("removing backup group {:?}", full_path);
+
+        // remove all individual backup dirs first to ensure nothing is using them
+        for snap in backup_group.list_backups(&self.base_path())? {
+            self.remove_backup_dir(&snap.backup_dir, false)?;
+        }
+
+        // no snapshots left, we can now safely remove the empty folder
         std::fs::remove_dir_all(&full_path)
             .map_err(|err| {
                 format_err!(
-                    "removing backup group {:?} failed - {}",
+                    "removing backup group directory {:?} failed - {}",
                     full_path,
                     err,
                 )
@@ -225,9 +240,10 @@ impl DataStore {
 
         let full_path = self.snapshot_path(backup_dir);
 
-        let _guard;
+        let (_guard, _manifest_guard);
         if !force {
-            _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or used as base")?;
+            _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or in use")?;
+            _manifest_guard = self.lock_manifest(backup_dir);
         }
 
         log::info!("removing backup snapshot {:?}", full_path);
@@ -260,8 +276,8 @@ impl DataStore {
 
     /// Returns the backup owner.
     ///
-    /// The backup owner is the user who first created the backup group.
-    pub fn get_owner(&self, backup_group: &BackupGroup) -> Result<Userid, Error> {
+    /// The backup owner is the entity who first created the backup group.
+    pub fn get_owner(&self, backup_group: &BackupGroup) -> Result<Authid, Error> {
         let mut full_path = self.base_path();
         full_path.push(backup_group.group_path());
         full_path.push("owner");
@@ -273,7 +289,7 @@ impl DataStore {
     pub fn set_owner(
         &self,
         backup_group: &BackupGroup,
-        userid: &Userid,
+        auth_id: &Authid,
         force: bool,
     ) -> Result<(), Error> {
         let mut path = self.base_path();
@@ -293,7 +309,7 @@ impl DataStore {
         let mut file = open_options.open(&path)
             .map_err(|err| format_err!("unable to create owner file {:?} - {}", path, err))?;
 
-        write!(file, "{}\n", userid)
+        writeln!(file, "{}", auth_id)
             .map_err(|err| format_err!("unable to write owner file  {:?} - {}", path, err))?;
 
         Ok(())
@@ -308,8 +324,8 @@ impl DataStore {
     pub fn create_locked_backup_group(
         &self,
         backup_group: &BackupGroup,
-        userid: &Userid,
-    ) -> Result<(Userid, DirLockGuard), Error> {
+        auth_id: &Authid,
+    ) -> Result<(Authid, DirLockGuard), Error> {
         // create intermediate path first:
         let base_path = self.base_path();
 
@@ -323,7 +339,7 @@ impl DataStore {
         match std::fs::create_dir(&full_path) {
             Ok(_) => {
                 let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?;
-                self.set_owner(backup_group, userid, false)?;
+                self.set_owner(backup_group, auth_id, false)?;
                 let owner = self.get_owner(backup_group)?; // just to be sure
                 Ok((owner, guard))
             }
@@ -442,26 +458,35 @@ impl DataStore {
     ) -> Result<(), Error> {
 
         let image_list = self.list_images()?;
-
         let image_count = image_list.len();
 
         let mut done = 0;
-
         let mut last_percentage: usize = 0;
 
-        for path in image_list {
+        for img in image_list {
 
             worker.check_abort()?;
             tools::fail_on_shutdown()?;
 
-            if let Ok(archive_type) = archive_type(&path) {
-                if archive_type == ArchiveType::FixedIndex {
-                    let index = self.open_fixed_reader(&path)?;
-                    self.index_mark_used_chunks(index, &path, status, worker)?;
-                } else if archive_type == ArchiveType::DynamicIndex {
-                    let index = self.open_dynamic_reader(&path)?;
-                    self.index_mark_used_chunks(index, &path, status, worker)?;
+            let path = self.chunk_store.relative_path(&img);
+            match std::fs::File::open(&path) {
+                Ok(file) => {
+                    if let Ok(archive_type) = archive_type(&img) {
+                        if archive_type == ArchiveType::FixedIndex {
+                            let index = FixedIndexReader::new(file).map_err(|e| {
+                                format_err!("can't read index '{}' - {}", path.to_string_lossy(), e)
+                            })?;
+                            self.index_mark_used_chunks(index, &img, status, worker)?;
+                        } else if archive_type == ArchiveType::DynamicIndex {
+                            let index = DynamicIndexReader::new(file).map_err(|e| {
+                                format_err!("can't read index '{}' - {}", path.to_string_lossy(), e)
+                            })?;
+                            self.index_mark_used_chunks(index, &img, status, worker)?;
+                        }
+                    }
                 }
+                Err(err) if err.kind() == io::ErrorKind::NotFound => (), // ignore vanished files
+                Err(err) => bail!("can't open index {} - {}", path.to_string_lossy(), err),
             }
             done += 1;
 
@@ -531,7 +556,11 @@ impl DataStore {
                 );
             }
             if gc_status.removed_bad > 0 {
-                crate::task_log!(worker, "Removed bad files: {}", gc_status.removed_bad);
+                crate::task_log!(worker, "Removed bad chunks: {}", gc_status.removed_bad);
+            }
+
+            if gc_status.still_bad > 0 {
+                crate::task_log!(worker, "Leftover bad chunks: {}", gc_status.still_bad);
             }
 
             crate::task_log!(
@@ -552,11 +581,36 @@ impl DataStore {
 
             crate::task_log!(worker, "On-Disk chunks: {}", gc_status.disk_chunks);
 
+            let deduplication_factor = if gc_status.disk_bytes > 0 {
+                (gc_status.index_data_bytes as f64)/(gc_status.disk_bytes as f64)
+            } else {
+                1.0
+            };
+
+            crate::task_log!(worker, "Deduplication factor: {:.2}", deduplication_factor);
+
             if gc_status.disk_chunks > 0 {
                 let avg_chunk = gc_status.disk_bytes/(gc_status.disk_chunks as u64);
                 crate::task_log!(worker, "Average chunk size: {}", HumanByte::from(avg_chunk));
             }
 
+            if let Ok(serialized) = serde_json::to_string(&gc_status) {
+                let mut path = self.base_path();
+                path.push(".gc-status");
+
+                let backup_user = crate::backup::backup_user()?;
+                let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644);
+                // set the correct owner/group/permissions while saving file
+                // owner(rw) = backup, group(r)= backup
+                let options = CreateOptions::new()
+                    .perm(mode)
+                    .owner(backup_user.uid)
+                    .group(backup_user.gid);
+
+                // ignore errors
+                let _ = replace_file(path, serialized.as_bytes(), options);
+            }
+
             *self.last_gc_status.lock().unwrap() = gc_status;
 
         } else {
@@ -611,8 +665,27 @@ impl DataStore {
             digest_str,
             err,
         ))
-     }
+    }
 
+    fn lock_manifest(
+        &self,
+        backup_dir: &BackupDir,
+    ) -> Result<File, Error> {
+        let mut path = self.base_path();
+        path.push(backup_dir.relative_path());
+        path.push(&MANIFEST_LOCK_NAME);
+
+        // update_manifest should never take a long time, so if someone else has
+        // the lock we can simply block a bit and should get it soon
+        open_file_locked(&path, Duration::from_secs(5), true)
+            .map_err(|err| {
+                format_err!(
+                    "unable to acquire manifest lock {:?} - {}", &path, err
+                )
+            })
+    }
+
+    /// Load the manifest without a lock. Must not be written back.
     pub fn load_manifest(
         &self,
         backup_dir: &BackupDir,
@@ -623,22 +696,20 @@ impl DataStore {
         Ok((manifest, raw_size))
     }
 
-    pub fn load_manifest_json(
+    /// Update the manifest of the specified snapshot. Never write a manifest directly,
+    /// only use this method - anything else may break locking guarantees.
+    pub fn update_manifest(
         &self,
         backup_dir: &BackupDir,
-    ) -> Result<Value, Error> {
-        let blob = self.load_blob(backup_dir, MANIFEST_BLOB_NAME)?;
-        // no expected digest available
-        let manifest_data = blob.decode(None, None)?;
-        let manifest: Value = serde_json::from_slice(&manifest_data[..])?;
-        Ok(manifest)
-    }
-
-    pub fn store_manifest(
-        &self,
-        backup_dir: &BackupDir,
-        manifest: Value,
+        update_fn: impl FnOnce(&mut BackupManifest),
     ) -> Result<(), Error> {
+
+        let _guard = self.lock_manifest(backup_dir)?;
+        let (mut manifest, _) = self.load_manifest(&backup_dir)?;
+
+        update_fn(&mut manifest);
+
+        let manifest = serde_json::to_value(manifest)?;
         let manifest = serde_json::to_string_pretty(&manifest)?;
         let blob = DataBlob::encode(manifest.as_bytes(), None, true)?;
         let raw_data = blob.raw_data();
@@ -647,8 +718,13 @@ impl DataStore {
         path.push(backup_dir.relative_path());
         path.push(MANIFEST_BLOB_NAME);
 
+        // atomic replace invalidates flock - no other writes past this point!
         replace_file(&path, raw_data, CreateOptions::new())?;
 
         Ok(())
     }
+
+    pub fn verify_new(&self) -> bool {
+        self.verify_new
+    }
 }

src/backup/dynamic_index.rs

@@ -95,6 +95,18 @@ impl DynamicIndexReader {
 
         let header_size = std::mem::size_of::<DynamicIndexHeader>();
 
+        let rawfd = file.as_raw_fd();
+        let stat = match nix::sys::stat::fstat(rawfd) {
+            Ok(stat) => stat,
+            Err(err) => bail!("fstat failed - {}", err),
+        };
+
+        let size = stat.st_size as usize;
+
+        if size < header_size {
+            bail!("index too small ({})", stat.st_size);
+        }
+
         let header: Box<DynamicIndexHeader> = unsafe { file.read_host_value_boxed()? };
 
         if header.magic != super::DYNAMIC_SIZED_CHUNK_INDEX_1_0 {
@@ -103,13 +115,7 @@ impl DynamicIndexReader {
 
         let ctime = proxmox::tools::time::epoch_i64();
 
-        let rawfd = file.as_raw_fd();
-
-        let stat = nix::sys::stat::fstat(rawfd)?;
-
-        let size = stat.st_size as usize;
-
-        let index_size = size - header_size;
+        let index_size = stat.st_size as usize - header_size;
         let index_count = index_size / 40;
         if index_count * 40 != index_size {
             bail!("got unexpected file size");

src/backup/fixed_index.rs

@@ -68,6 +68,19 @@ impl FixedIndexReader {
         file.seek(SeekFrom::Start(0))?;
 
         let header_size = std::mem::size_of::<FixedIndexHeader>();
+
+        let rawfd = file.as_raw_fd();
+        let stat = match nix::sys::stat::fstat(rawfd) {
+            Ok(stat) => stat,
+            Err(err) => bail!("fstat failed - {}", err),
+        };
+
+        let size = stat.st_size as usize;
+
+        if size < header_size {
+            bail!("index too small ({})", stat.st_size);
+        }
+
         let header: Box<FixedIndexHeader> = unsafe { file.read_host_value_boxed()? };
 
         if header.magic != super::FIXED_SIZED_CHUNK_INDEX_1_0 {
@@ -81,12 +94,6 @@ impl FixedIndexReader {
         let index_length = ((size + chunk_size - 1) / chunk_size) as usize;
         let index_size = index_length * 32;
 
-        let rawfd = file.as_raw_fd();
-
-        let stat = match nix::sys::stat::fstat(rawfd) {
-            Ok(stat) => stat,
-            Err(err) => bail!("fstat failed - {}", err),
-        };
 
         let expected_index_size = (stat.st_size as usize) - header_size;
         if index_size != expected_index_size {

src/backup/manifest.rs

@@ -8,6 +8,7 @@ use ::serde::{Deserialize, Serialize};
 use crate::backup::{BackupDir, CryptMode, CryptConfig};
 
 pub const MANIFEST_BLOB_NAME: &str = "index.json.blob";
+pub const MANIFEST_LOCK_NAME: &str = ".index.json.lck";
 pub const CLIENT_LOG_BLOB_NAME: &str = "client.log.blob";
 
 mod hex_csum {

src/backup/verify.rs

@@ -2,6 +2,7 @@ use std::collections::HashSet;
 use std::sync::{Arc, Mutex};
 use std::sync::atomic::{Ordering, AtomicUsize};
 use std::time::Instant;
+use nix::dir::Dir;
 
 use anyhow::{bail, format_err, Error};
 
@@ -13,6 +14,7 @@ use crate::{
         BackupGroup,
         BackupDir,
         BackupInfo,
+        BackupManifest,
         IndexFile,
         CryptMode,
         FileInfo,
@@ -23,6 +25,7 @@ use crate::{
     task::TaskState,
     task_log,
     tools::ParallelHandler,
+    tools::fs::lock_dir_noblock_shared,
 };
 
 fn verify_blob(datastore: Arc<DataStore>, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
@@ -282,9 +285,48 @@ pub fn verify_backup_dir(
     corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
     worker: Arc<dyn TaskState + Send + Sync>,
     upid: UPID,
+    filter: Option<&dyn Fn(&BackupManifest) -> bool>,
 ) -> Result<bool, Error> {
+    let snap_lock = lock_dir_noblock_shared(
+        &datastore.snapshot_path(&backup_dir),
+        "snapshot",
+        "locked by another operation");
+    match snap_lock {
+        Ok(snap_lock) => verify_backup_dir_with_lock(
+            datastore,
+            backup_dir,
+            verified_chunks,
+            corrupt_chunks,
+            worker,
+            upid,
+            filter,
+            snap_lock
+        ),
+        Err(err) => {
+            task_log!(
+                worker,
+                "SKIPPED: verify {}:{} - could not acquire snapshot lock: {}",
+                datastore.name(),
+                backup_dir,
+                err,
+            );
+            Ok(true)
+        }
+    }
+}
 
-    let mut manifest = match datastore.load_manifest(&backup_dir) {
+/// See verify_backup_dir
+pub fn verify_backup_dir_with_lock(
+    datastore: Arc<DataStore>,
+    backup_dir: &BackupDir,
+    verified_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    corrupt_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
+    worker: Arc<dyn TaskState + Send + Sync>,
+    upid: UPID,
+    filter: Option<&dyn Fn(&BackupManifest) -> bool>,
+    _snap_lock: Dir,
+) -> Result<bool, Error> {
+    let manifest = match datastore.load_manifest(&backup_dir) {
         Ok((manifest, _)) => manifest,
         Err(err) => {
             task_log!(
@@ -298,6 +340,18 @@ pub fn verify_backup_dir(
         }
     };
 
+    if let Some(filter) = filter {
+        if filter(&manifest) == false {
+            task_log!(
+                worker,
+                "SKIPPED: verify {}:{} (recently verified)",
+                datastore.name(),
+                backup_dir,
+            );
+            return Ok(true);
+        }
+    }
+
     task_log!(worker, "verify {}:{}", datastore.name(), backup_dir);
 
     let mut error_count = 0;
@@ -351,9 +405,10 @@ pub fn verify_backup_dir(
         state: verify_result,
         upid,
     };
-    manifest.unprotected["verify_state"] = serde_json::to_value(verify_state)?;
-    datastore.store_manifest(&backup_dir, serde_json::to_value(manifest)?)
-        .map_err(|err| format_err!("unable to store manifest blob - {}", err))?;
+    let verify_state = serde_json::to_value(verify_state)?;
+    datastore.update_manifest(&backup_dir, |manifest| {
+        manifest.unprotected["verify_state"] = verify_state;
+    }).map_err(|err| format_err!("unable to update manifest blob - {}", err))?;
 
     Ok(error_count == 0)
 }
@@ -373,6 +428,7 @@ pub fn verify_backup_group(
     progress: Option<(usize, usize)>, // (done, snapshot_count)
     worker: Arc<dyn TaskState + Send + Sync>,
     upid: &UPID,
+    filter: Option<&dyn Fn(&BackupManifest) -> bool>,
 ) -> Result<(usize, Vec<String>), Error> {
 
     let mut errors = Vec::new();
@@ -398,6 +454,7 @@ pub fn verify_backup_group(
     BackupInfo::sort_list(&mut list, false); // newest first
     for info in list {
         count += 1;
+
         if !verify_backup_dir(
             datastore.clone(),
             &info.backup_dir,
@@ -405,6 +462,7 @@ pub fn verify_backup_group(
             corrupt_chunks.clone(),
             worker.clone(),
             upid.clone(),
+            filter,
         )? {
             errors.push(info.backup_dir.to_string());
         }
@@ -424,7 +482,7 @@ pub fn verify_backup_group(
     Ok((count, errors))
 }
 
-/// Verify all backups inside a datastore
+/// Verify all (owned) backups inside a datastore
 ///
 /// Errors are logged to the worker log.
 ///
@@ -435,13 +493,41 @@ pub fn verify_all_backups(
     datastore: Arc<DataStore>,
     worker: Arc<dyn TaskState + Send + Sync>,
     upid: &UPID,
+    owner: Option<Authid>,
+    filter: Option<&dyn Fn(&BackupManifest) -> bool>,
 ) -> Result<Vec<String>, Error> {
     let mut errors = Vec::new();
 
+    if let Some(owner) = &owner {
+        task_log!(
+            worker,
+            "verify datastore {} - limiting to backups owned by {}",
+            datastore.name(),
+            owner
+        );
+    }
+
+    let filter_by_owner = |group: &BackupGroup| {
+        if let Some(owner) = &owner {
+            match datastore.get_owner(group) {
+                Ok(ref group_owner) => {
+                    group_owner == owner
+                        || (group_owner.is_token()
+                            && !owner.is_token()
+                            && group_owner.user() == owner.user())
+                },
+                Err(_) => false,
+            }
+        } else {
+            true
+        }
+    };
+
     let mut list = match BackupGroup::list_groups(&datastore.base_path()) {
         Ok(list) => list
             .into_iter()
             .filter(|group| !(group.backup_type() == "host" && group.backup_id() == "benchmark"))
+            .filter(filter_by_owner)
             .collect::<Vec<BackupGroup>>(),
         Err(err) => {
             task_log!(
@@ -479,6 +565,7 @@ pub fn verify_all_backups(
             Some((done, snapshot_count)),
             worker.clone(),
             upid,
+            filter,
         )?;
         errors.append(&mut group_errors);
 

src/bin/proxmox-backup-api.rs

@@ -37,7 +37,7 @@ async fn run() -> Result<(), Error> {
     config::update_self_signed_cert(false)?;
 
     proxmox_backup::rrd::create_rrdb_dir()?;
-    proxmox_backup::config::jobstate::create_jobstate_dir()?;
+    proxmox_backup::server::jobstate::create_jobstate_dir()?;
 
     if let Err(err) = generate_auth_key() {
         bail!("unable to generate auth key - {}", err);
@@ -49,9 +49,13 @@ async fn run() -> Result<(), Error> {
     }
     let _ = csrf_secret(); // load with lazy_static
 
-    let config = server::ApiConfig::new(
+    let mut config = server::ApiConfig::new(
         buildcfg::JS_DIR, &proxmox_backup::api2::ROUTER, RpcEnvironmentType::PRIVILEGED)?;
 
+    let mut commando_sock = server::CommandoSocket::new(server::our_ctrl_sock());
+
+    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN, &mut commando_sock)?;
+
     let rest_server = RestServer::new(config);
 
     // http server future:
@@ -74,10 +78,12 @@ async fn run() -> Result<(), Error> {
         },
     );
 
+    server::write_pid(buildcfg::PROXMOX_BACKUP_API_PID_FN)?;
     daemon::systemd_notify(daemon::SystemdNotify::Ready)?;
 
     let init_result: Result<(), Error> = try_block!({
-        server::create_task_control_socket()?;
+        server::register_task_control_commands(&mut commando_sock)?;
+        commando_sock.spawn()?;
         server::server_state_init()?;
         Ok(())
     });

src/bin/proxmox-backup-client.rs

@@ -32,11 +32,11 @@ use proxmox::{
 use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
 
 use proxmox_backup::tools;
+use proxmox_backup::api2::access::user::UserWithTokens;
 use proxmox_backup::api2::types::*;
 use proxmox_backup::api2::version;
 use proxmox_backup::client::*;
 use proxmox_backup::pxar::catalog::*;
-use proxmox_backup::config::user::complete_user_name;
 use proxmox_backup::backup::{
     archive_type,
     decrypt_key,
@@ -193,7 +193,7 @@ pub fn complete_repository(_arg: &str, _param: &HashMap<String, String>) -> Vec<
     result
 }
 
-fn connect(server: &str, port: u16, userid: &Userid) -> Result<HttpClient, Error> {
+fn connect(server: &str, port: u16, auth_id: &Authid) -> Result<HttpClient, Error> {
 
     let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok();
 
@@ -212,7 +212,7 @@ fn connect(server: &str, port: u16, userid: &Userid) -> Result<HttpClient, Error
         .fingerprint_cache(true)
         .ticket_cache(true);
 
-    HttpClient::new(server, port, userid, options)
+    HttpClient::new(server, port, auth_id, options)
 }
 
 async fn view_task_result(
@@ -366,7 +366,7 @@ async fn list_backup_groups(param: Value) -> Result<Value, Error> {
 
     let repo = extract_repository_from_value(&param)?;
 
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let path = format!("api2/json/admin/datastore/{}/groups", repo.store());
 
@@ -425,7 +425,7 @@ async fn list_backup_groups(param: Value) -> Result<Value, Error> {
                 description: "Backup group.",
             },
             "new-owner": {
-                type: Userid,
+                type: Authid,
             },
         }
    }
@@ -435,7 +435,7 @@ async fn change_backup_owner(group: String, mut param: Value) -> Result<(), Erro
 
     let repo = extract_repository_from_value(&param)?;
 
-    let mut client = connect(repo.host(), repo.port(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     param.as_object_mut().unwrap().remove("repository");
 
@@ -478,7 +478,7 @@ async fn list_snapshots(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let group: Option<BackupGroup> = if let Some(path) = param["group"].as_str() {
         Some(path.parse()?)
@@ -510,7 +510,7 @@ async fn list_snapshots(param: Value) -> Result<Value, Error> {
         .sortby("backup-id", false)
         .sortby("backup-time", false)
         .column(ColumnConfig::new("backup-id").renderer(render_snapshot_path).header("snapshot"))
-        .column(ColumnConfig::new("size"))
+        .column(ColumnConfig::new("size").renderer(tools::format::render_bytes_human_readable))
         .column(ColumnConfig::new("files").renderer(render_files))
         ;
 
@@ -543,7 +543,7 @@ async fn forget_snapshots(param: Value) -> Result<Value, Error> {
     let path = tools::required_string_param(&param, "snapshot")?;
     let snapshot: BackupDir = path.parse()?;
 
-    let mut client = connect(repo.host(), repo.port(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let path = format!("api2/json/admin/datastore/{}/snapshots", repo.store());
 
@@ -573,7 +573,7 @@ async fn api_login(param: Value) -> Result<Value, Error> {
 
     let repo = extract_repository_from_value(&param)?;
 
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
     client.login().await?;
 
     record_repository(&repo);
@@ -630,7 +630,7 @@ async fn api_version(param: Value) -> Result<(), Error> {
 
     let repo = extract_repository_from_value(&param);
     if let Ok(repo) = repo {
-        let client = connect(repo.host(), repo.port(), repo.user())?;
+        let client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
         match client.get("api2/json/version", None).await {
             Ok(mut result) => version_info["server"] = result["data"].take(),
@@ -680,7 +680,7 @@ async fn list_snapshot_files(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let path = format!("api2/json/admin/datastore/{}/files", repo.store());
 
@@ -724,7 +724,7 @@ async fn start_garbage_collection(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let mut client = connect(repo.host(), repo.port(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let path = format!("api2/json/admin/datastore/{}/gc", repo.store());
 
@@ -1036,7 +1036,7 @@ async fn create_backup(
 
     let backup_time = backup_time_opt.unwrap_or_else(|| epoch_i64());
 
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
     record_repository(&repo);
 
     println!("Starting backup: {}/{}/{}", backup_type, backup_id, BackupDir::backup_time_to_string(backup_time)?);
@@ -1339,7 +1339,7 @@ async fn restore(param: Value) -> Result<Value, Error> {
 
     let archive_name = tools::required_string_param(&param, "archive-name")?;
 
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     record_repository(&repo);
 
@@ -1512,7 +1512,7 @@ async fn upload_log(param: Value) -> Result<Value, Error> {
     let snapshot = tools::required_string_param(&param, "snapshot")?;
     let snapshot: BackupDir = snapshot.parse()?;
 
-    let mut client = connect(repo.host(), repo.port(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let (keydata, crypt_mode) = keyfile_parameters(&param)?;
 
@@ -1583,7 +1583,7 @@ fn prune<'a>(
 async fn prune_async(mut param: Value) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
 
-    let mut client = connect(repo.host(), repo.port(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let path = format!("api2/json/admin/datastore/{}/prune", repo.store());
 
@@ -1657,7 +1657,10 @@ async fn prune_async(mut param: Value) -> Result<Value, Error> {
                optional: true,
            },
        }
-   }
+   },
+    returns: {
+        type: StorageStatus,
+    },
 )]
 /// Get repository status.
 async fn status(param: Value) -> Result<Value, Error> {
@@ -1666,7 +1669,7 @@ async fn status(param: Value) -> Result<Value, Error> {
 
     let output_format = get_output_format(&param);
 
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let path = format!("api2/json/admin/datastore/{}/status", repo.store());
 
@@ -1690,7 +1693,7 @@ async fn status(param: Value) -> Result<Value, Error> {
         .column(ColumnConfig::new("used").renderer(render_total_percentage))
         .column(ColumnConfig::new("avail").renderer(render_total_percentage));
 
-    let schema = &proxmox_backup::api2::admin::datastore::API_RETURN_SCHEMA_STATUS;
+    let schema = &API_RETURN_SCHEMA_STATUS;
 
     format_and_print_result_full(&mut data, schema, &output_format, &options);
 
@@ -1711,7 +1714,7 @@ async fn try_get(repo: &BackupRepository, url: &str) -> Value {
         .fingerprint_cache(true)
         .ticket_cache(true);
 
-    let client = match HttpClient::new(repo.host(), repo.port(), repo.user(), options) {
+    let client = match HttpClient::new(repo.host(), repo.port(), repo.auth_id(), options) {
         Ok(v) => v,
         _ => return Value::Null,
     };
@@ -1901,6 +1904,33 @@ fn complete_chunk_size(_arg: &str, _param: &HashMap<String, String>) -> Vec<Stri
     result
 }
 
+fn complete_auth_id(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
+    proxmox_backup::tools::runtime::main(async { complete_auth_id_do(param).await })
+}
+
+async fn complete_auth_id_do(param: &HashMap<String, String>) -> Vec<String> {
+
+    let mut result = vec![];
+
+    let repo = match extract_repository_from_map(param) {
+        Some(v) => v,
+        _ => return result,
+    };
+
+    let data = try_get(&repo, "api2/json/access/users?include_tokens=true").await;
+
+    if let Ok(parsed) = serde_json::from_value::<Vec<UserWithTokens>>(data) {
+        for user in parsed {
+            result.push(user.userid.to_string());
+            for token in user.tokens {
+                result.push(token.tokenid.to_string());
+            }
+        }
+    };
+
+    result
+}
+
 use proxmox_backup::client::RemoteChunkReader;
 /// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
 /// async use!
@@ -2010,7 +2040,7 @@ fn main() {
     let change_owner_cmd_def = CliCommand::new(&API_METHOD_CHANGE_BACKUP_OWNER)
         .arg_param(&["group", "new-owner"])
         .completion_cb("group", complete_backup_group)
-        .completion_cb("new-owner",  complete_user_name)
+        .completion_cb("new-owner",  complete_auth_id)
         .completion_cb("repository", complete_repository);
 
     let cmd_def = CliCommandMap::new()

src/bin/proxmox-backup-manager.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::io::{self, Write};
 
 use anyhow::{format_err, Error};
 use serde_json::{json, Value};
@@ -62,10 +63,10 @@ fn connect() -> Result<HttpClient, Error> {
         let ticket = Ticket::new("PBS", Userid::root_userid())?
             .sign(private_auth_key(), None)?;
         options = options.password(Some(ticket));
-        HttpClient::new("localhost", 8007, Userid::root_userid(), options)?
+        HttpClient::new("localhost", 8007, Authid::root_auth_id(), options)?
     } else {
         options = options.ticket_cache(true).interactive(true);
-        HttpClient::new("localhost", 8007, Userid::root_userid(), options)?
+        HttpClient::new("localhost", 8007, Authid::root_auth_id(), options)?
     };
 
     Ok(client)
@@ -354,6 +355,14 @@ async fn verify(
     Ok(Value::Null)
 }
 
+#[api()]
+/// System report
+async fn report() -> Result<Value, Error> {
+    let report = proxmox_backup::server::generate_report();
+    io::stdout().write_all(report.as_bytes())?;
+    Ok(Value::Null)
+}
+
 fn main() {
 
     proxmox_backup::tools::setup_safe_path_env();
@@ -368,6 +377,7 @@ fn main() {
         .insert("remote", remote_commands())
         .insert("garbage-collection", garbage_collection_commands())
         .insert("cert", cert_mgmt_cli())
+        .insert("subscription", subscription_commands())
         .insert("sync-job", sync_job_commands())
         .insert("task", task_mgmt_cli())
         .insert(
@@ -383,12 +393,15 @@ fn main() {
             CliCommand::new(&API_METHOD_VERIFY)
                 .arg_param(&["store"])
                 .completion_cb("store", config::datastore::complete_datastore_name)
+        )
+        .insert("report",
+            CliCommand::new(&API_METHOD_REPORT)
         );
 
 
 
     let mut rpcenv = CliEnvironment::new();
-    rpcenv.set_user(Some(String::from("root@pam")));
+    rpcenv.set_auth_id(Some(String::from("root@pam")));
 
    proxmox_backup::tools::runtime::main(run_async_cli_command(cmd_def, rpcenv));
 }
@@ -400,29 +413,13 @@ pub fn complete_remote_datastore_name(_arg: &str, param: &HashMap<String, String
 
     let _ = proxmox::try_block!({
         let remote = param.get("remote").ok_or_else(|| format_err!("no remote"))?;
-        let (remote_config, _digest) = config::remote::config()?;
 
-        let remote: config::remote::Remote = remote_config.lookup("remote", &remote)?;
+        let data = crate::tools::runtime::block_on(async move {
+            crate::api2::config::remote::scan_remote_datastores(remote.clone()).await
+        })?;
 
-        let options = HttpClientOptions::new()
-            .password(Some(remote.password.clone()))
-            .fingerprint(remote.fingerprint.clone());
-
-        let client = HttpClient::new(
-            &remote.host,
-            remote.port.unwrap_or(8007),
-            &remote.userid,
-            options,
-        )?;
-
-        let result = crate::tools::runtime::block_on(client.get("api2/json/admin/datastore", None))?;
-
-        if let Some(data) = result["data"].as_array() {
-            for item in data {
-                if let Some(store) = item["store"].as_str() {
-                    list.push(store.to_owned());
-                }
-            }
+        for item in data {
+            list.push(item.store);
         }
 
         Ok(())

src/bin/proxmox-backup-proxy.rs

@@ -1,5 +1,6 @@
-use std::sync::{Arc};
+use std::sync::Arc;
 use std::path::{Path, PathBuf};
+use std::os::unix::io::AsRawFd;
 
 use anyhow::{bail, format_err, Error};
 use futures::*;
@@ -9,16 +10,46 @@ use openssl::ssl::{SslMethod, SslAcceptor, SslFiletype};
 use proxmox::try_block;
 use proxmox::api::RpcEnvironmentType;
 
-use proxmox_backup::api2::types::Userid;
+use proxmox_backup::{
+    backup::DataStore,
+    server::{
+        WorkerTask,
+        ApiConfig,
+        rest::*,
+        jobstate::{
+            self,
+            Job,
+        },
+        rotate_task_log_archive,
+    },
+    tools::systemd::time::{
+        parse_calendar_event,
+        compute_next_event,
+    },
+};
+
+
+use proxmox_backup::api2::types::Authid;
 use proxmox_backup::configdir;
 use proxmox_backup::buildcfg;
 use proxmox_backup::server;
-use proxmox_backup::tools::daemon;
-use proxmox_backup::server::{ApiConfig, rest::*};
 use proxmox_backup::auth_helpers::*;
-use proxmox_backup::tools::disks::{ DiskManage, zfs_pool_stats };
+use proxmox_backup::tools::{
+    daemon,
+    disks::{
+        DiskManage,
+        zfs_pool_stats,
+    },
+    logrotate::LogRotate,
+    socket::{
+        set_tcp_keepalive,
+        PROXMOX_BACKUP_TCP_KEEPALIVE_TIME,
+    },
+};
 
 use proxmox_backup::api2::pull::do_sync_job;
+use proxmox_backup::server::do_verification_job;
+use proxmox_backup::server::do_prune_job;
 
 fn main() -> Result<(), Error> {
     proxmox_backup::tools::setup_safe_path_env();
@@ -43,6 +74,10 @@ async fn run() -> Result<(), Error> {
         bail!("unable to inititialize syslog - {}", err);
     }
 
+    // Note: To debug early connection error use
+    // PROXMOX_DEBUG=1 ./target/release/proxmox-backup-proxy
+    let debug = std::env::var("PROXMOX_DEBUG").is_ok();
+
     let _ = public_auth_key(); // load with lazy_static
     let _ = csrf_secret(); // load with lazy_static
 
@@ -55,7 +90,6 @@ async fn run() -> Result<(), Error> {
     config.add_alias("xtermjs", "/usr/share/pve-xtermjs");
     config.add_alias("locale", "/usr/share/pbs-i18n");
     config.add_alias("widgettoolkit", "/usr/share/javascript/proxmox-widget-toolkit");
-    config.add_alias("css", "/usr/share/javascript/proxmox-backup/css");
     config.add_alias("docs", "/usr/share/doc/proxmox-backup/html");
 
     let mut indexpath = PathBuf::from(buildcfg::JS_DIR);
@@ -63,6 +97,10 @@ async fn run() -> Result<(), Error> {
     config.register_template("index", &indexpath)?;
     config.register_template("console", "/usr/share/pve-xtermjs/index.html.hbs")?;
 
+    let mut commando_sock = server::CommandoSocket::new(server::our_ctrl_sock());
+
+    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN, &mut commando_sock)?;
+
     let rest_server = RestServer::new(config);
 
     //openssl req -x509 -newkey rsa:4096 -keyout /etc/proxmox-backup/proxy.key -out /etc/proxmox-backup/proxy.pem -nodes
@@ -81,22 +119,12 @@ async fn run() -> Result<(), Error> {
     let server = daemon::create_daemon(
         ([0,0,0,0,0,0,0,0], 8007).into(),
         |listener, ready| {
-            let connections = proxmox_backup::tools::async_io::StaticIncoming::from(listener)
-                .map_err(Error::from)
-                .try_filter_map(move |(sock, _addr)| {
-                    let acceptor = Arc::clone(&acceptor);
-                    async move {
-                        sock.set_nodelay(true).unwrap();
-                        Ok(tokio_openssl::accept(&acceptor, sock)
-                            .await
-                            .ok() // handshake errors aren't be fatal, so return None to filter
-                        )
-                    }
-                });
-            let connections = proxmox_backup::tools::async_io::HyperAccept(connections);
+
+            let connections = accept_connections(listener, acceptor, debug);
+            let connections = hyper::server::accept::from_stream(connections);
 
             Ok(ready
-                .and_then(|_| hyper::Server::builder(connections)
+               .and_then(|_| hyper::Server::builder(connections)
                     .serve(rest_server)
                     .with_graceful_shutdown(server::shutdown_future())
                     .map_err(Error::from)
@@ -107,10 +135,12 @@ async fn run() -> Result<(), Error> {
         },
     );
 
+    server::write_pid(buildcfg::PROXMOX_BACKUP_PROXY_PID_FN)?;
     daemon::systemd_notify(daemon::SystemdNotify::Ready)?;
 
     let init_result: Result<(), Error> = try_block!({
-        server::create_task_control_socket()?;
+        server::register_task_control_commands(&mut commando_sock)?;
+        commando_sock.spawn()?;
         server::server_state_init()?;
         Ok(())
     });
@@ -130,6 +160,72 @@ async fn run() -> Result<(), Error> {
     Ok(())
 }
 
+fn accept_connections(
+    mut listener: tokio::net::TcpListener,
+    acceptor: Arc<openssl::ssl::SslAcceptor>,
+    debug: bool,
+) -> tokio::sync::mpsc::Receiver<Result<tokio_openssl::SslStream<tokio::net::TcpStream>, Error>> {
+
+    const MAX_PENDING_ACCEPTS: usize = 1024;
+
+    let (sender, receiver) = tokio::sync::mpsc::channel(MAX_PENDING_ACCEPTS);
+
+    let accept_counter = Arc::new(());
+
+    tokio::spawn(async move {
+        loop {
+            match listener.accept().await {
+                Err(err) => {
+                    eprintln!("error accepting tcp connection: {}", err);
+                }
+                Ok((sock, _addr)) =>  {
+                    sock.set_nodelay(true).unwrap();
+                    let _ = set_tcp_keepalive(sock.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME);
+                    let acceptor = Arc::clone(&acceptor);
+                    let mut sender = sender.clone();
+
+                    if Arc::strong_count(&accept_counter) > MAX_PENDING_ACCEPTS {
+                        eprintln!("connection rejected - to many open connections");
+                        continue;
+                    }
+
+                    let accept_counter = accept_counter.clone();
+                    tokio::spawn(async move {
+                        let accept_future = tokio::time::timeout(
+                            Duration::new(10, 0), tokio_openssl::accept(&acceptor, sock));
+
+                        let result = accept_future.await;
+
+                        match result {
+                            Ok(Ok(connection)) => {
+                                if let Err(_) = sender.send(Ok(connection)).await {
+                                    if debug {
+                                        eprintln!("detect closed connection channel");
+                                    }
+                                }
+                            }
+                            Ok(Err(err)) => {
+                                if debug {
+                                    eprintln!("https handshake failed - {}", err);
+                                }
+                            }
+                            Err(_) => {
+                                if debug {
+                                    eprintln!("https handshake timeout");
+                                }
+                            }
+                        }
+
+                        drop(accept_counter); // decrease reference count
+                    });
+                }
+            }
+        }
+    });
+
+    receiver
+}
+
 fn start_stat_generator() {
     let abort_future = server::shutdown_future();
     let future = Box::pin(run_stat_generator());
@@ -196,8 +292,8 @@ async fn schedule_tasks() -> Result<(), Error> {
 
     schedule_datastore_garbage_collection().await;
     schedule_datastore_prune().await;
-    schedule_datastore_verification().await;
     schedule_datastore_sync_jobs().await;
+    schedule_datastore_verify_jobs().await;
     schedule_task_log_rotate().await;
 
     Ok(())
@@ -205,14 +301,12 @@ async fn schedule_tasks() -> Result<(), Error> {
 
 async fn schedule_datastore_garbage_collection() {
 
-    use proxmox_backup::backup::DataStore;
-    use proxmox_backup::server::{UPID, WorkerTask};
     use proxmox_backup::config::{
-        jobstate::{self, Job},
-        datastore::{self, DataStoreConfig}
+        datastore::{
+            self,
+            DataStoreConfig,
+        },
     };
-    use proxmox_backup::tools::systemd::time::{
-        parse_calendar_event, compute_next_event};
 
     let config = match datastore::config() {
         Err(err) => {
@@ -256,22 +350,11 @@ async fn schedule_datastore_garbage_collection() {
 
         let worker_type = "garbage_collection";
 
-        let stat = datastore.last_gc_status();
-        let last = if let Some(upid_str) = stat.upid {
-            match upid_str.parse::<UPID>() {
-                Ok(upid) => upid.starttime,
-                Err(err) => {
-                    eprintln!("unable to parse upid '{}' - {}", upid_str, err);
-                    continue;
-                }
-            }
-        } else {
-            match jobstate::last_run_time(worker_type, &store) {
-                Ok(time) => time,
-                Err(err) => {
-                    eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
-                    continue;
-                }
+        let last = match jobstate::last_run_time(worker_type, &store) {
+            Ok(time) => time,
+            Err(err) => {
+                eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
+                continue;
             }
         };
 
@@ -288,51 +371,30 @@ async fn schedule_datastore_garbage_collection() {
 
         if next > now  { continue; }
 
-        let mut job = match Job::new(worker_type, &store) {
+        let job = match Job::new(worker_type, &store) {
             Ok(job) => job,
             Err(_) => continue, // could not get lock
         };
 
-        let store2 = store.clone();
+        let auth_id = Authid::root_auth_id();
 
-        if let Err(err) = WorkerTask::new_thread(
-            worker_type,
-            Some(store.clone()),
-            Userid::backup_userid().clone(),
-            false,
-            move |worker| {
-                job.start(&worker.upid().to_string())?;
-
-                worker.log(format!("starting garbage collection on store {}", store));
-                worker.log(format!("task triggered by schedule '{}'", event_str));
-
-                let result = datastore.garbage_collection(&*worker, worker.upid());
-
-                let status = worker.create_state(&result);
-
-                if let Err(err) = job.finish(status) {
-                    eprintln!("could not finish job state for {}: {}", worker_type, err);
-                }
-
-                result
-            }
-        ) {
-            eprintln!("unable to start garbage collection on store {} - {}", store2, err);
+        if let Err(err) = crate::server::do_garbage_collection_job(job, datastore, auth_id, Some(event_str), false) {
+            eprintln!("unable to start garbage collection job on datastore {} - {}", store, err);
         }
     }
 }
 
 async fn schedule_datastore_prune() {
 
-    use proxmox_backup::backup::{
-        PruneOptions, DataStore, BackupGroup, compute_prune_info};
-    use proxmox_backup::server::{WorkerTask};
-    use proxmox_backup::config::{
-        jobstate::{self, Job},
-        datastore::{self, DataStoreConfig}
+    use proxmox_backup::{
+        backup::{
+            PruneOptions,
+        },
+        config::datastore::{
+            self,
+            DataStoreConfig,
+        },
     };
-    use proxmox_backup::tools::systemd::time::{
-        parse_calendar_event, compute_next_event};
 
     let config = match datastore::config() {
         Err(err) => {
@@ -343,13 +405,6 @@ async fn schedule_datastore_prune() {
     };
 
     for (store, (_, store_config)) in config.sections {
-        let datastore = match DataStore::lookup_datastore(&store) {
-            Ok(datastore) => datastore,
-            Err(err) => {
-                eprintln!("lookup_datastore '{}' failed - {}", store, err);
-                continue;
-            }
-        };
 
         let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
             Ok(c) => c,
@@ -377,218 +432,26 @@ async fn schedule_datastore_prune() {
             continue;
         }
 
-        let event = match parse_calendar_event(&event_str) {
-            Ok(event) => event,
-            Err(err) => {
-                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
-                continue;
-            }
-        };
-
         let worker_type = "prune";
+        if check_schedule(worker_type, &event_str, &store) {
+            let job = match Job::new(worker_type, &store) {
+                Ok(job) => job,
+                Err(_) => continue, // could not get lock
+            };
 
-        let last = match jobstate::last_run_time(worker_type, &store) {
-            Ok(time) => time,
-            Err(err) => {
-                eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
-                continue;
+            let auth_id = Authid::root_auth_id().clone();
+            if let Err(err) = do_prune_job(job, prune_options, store.clone(), &auth_id, Some(event_str)) {
+                eprintln!("unable to start datastore prune job {} - {}", &store, err);
             }
         };
-
-        let next = match compute_next_event(&event, last, false) {
-            Ok(Some(next)) => next,
-            Ok(None) => continue,
-            Err(err) => {
-                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
-                continue;
-            }
-        };
-
-        let now = proxmox::tools::time::epoch_i64();
-
-        if next > now  { continue; }
-
-        let mut job = match Job::new(worker_type, &store) {
-            Ok(job) => job,
-            Err(_) => continue, // could not get lock
-        };
-
-        let store2 = store.clone();
-
-        if let Err(err) = WorkerTask::new_thread(
-            worker_type,
-            Some(store.clone()),
-            Userid::backup_userid().clone(),
-            false,
-            move |worker| {
-
-                job.start(&worker.upid().to_string())?;
-
-                let result = try_block!({
-
-                    worker.log(format!("Starting datastore prune on store \"{}\"", store));
-                    worker.log(format!("task triggered by schedule '{}'", event_str));
-                    worker.log(format!("retention options: {}", prune_options.cli_options_string()));
-
-                    let base_path = datastore.base_path();
-
-                    let groups = BackupGroup::list_groups(&base_path)?;
-                    for group in groups {
-                        let list = group.list_backups(&base_path)?;
-                        let mut prune_info = compute_prune_info(list, &prune_options)?;
-                        prune_info.reverse(); // delete older snapshots first
-
-                        worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
-                                store, group.backup_type(), group.backup_id()));
-
-                        for (info, keep) in prune_info {
-                            worker.log(format!(
-                                    "{} {}/{}/{}",
-                                    if keep { "keep" } else { "remove" },
-                                    group.backup_type(), group.backup_id(),
-                                    info.backup_dir.backup_time_string()));
-                            if !keep {
-                                datastore.remove_backup_dir(&info.backup_dir, true)?;
-                            }
-                        }
-                    }
-                    Ok(())
-                });
-
-                let status = worker.create_state(&result);
-
-                if let Err(err) = job.finish(status) {
-                    eprintln!("could not finish job state for {}: {}", worker_type, err);
-                }
-
-                result
-            }
-        ) {
-            eprintln!("unable to start datastore prune on store {} - {}", store2, err);
-        }
-    }
-}
-
-async fn schedule_datastore_verification() {
-    use proxmox_backup::backup::{DataStore, verify_all_backups};
-    use proxmox_backup::server::{WorkerTask};
-    use proxmox_backup::config::{
-        jobstate::{self, Job},
-        datastore::{self, DataStoreConfig}
-    };
-    use proxmox_backup::tools::systemd::time::{
-        parse_calendar_event, compute_next_event};
-
-    let config = match datastore::config() {
-        Err(err) => {
-            eprintln!("unable to read datastore config - {}", err);
-            return;
-        }
-        Ok((config, _digest)) => config,
-    };
-
-    for (store, (_, store_config)) in config.sections {
-        let datastore = match DataStore::lookup_datastore(&store) {
-            Ok(datastore) => datastore,
-            Err(err) => {
-                eprintln!("lookup_datastore failed - {}", err);
-                continue;
-            }
-        };
-
-        let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
-            Ok(c) => c,
-            Err(err) => {
-                eprintln!("datastore config from_value failed - {}", err);
-                continue;
-            }
-        };
-
-        let event_str = match store_config.verify_schedule {
-            Some(event_str) => event_str,
-            None => continue,
-        };
-
-        let event = match parse_calendar_event(&event_str) {
-            Ok(event) => event,
-            Err(err) => {
-                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
-                continue;
-            }
-        };
-
-        let worker_type = "verify";
-
-        let last = match jobstate::last_run_time(worker_type, &store) {
-            Ok(time) => time,
-            Err(err) => {
-                eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
-                continue;
-            }
-        };
-
-        let next = match compute_next_event(&event, last, false) {
-            Ok(Some(next)) => next,
-            Ok(None) => continue,
-            Err(err) => {
-                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
-                continue;
-            }
-        };
-
-        let now = proxmox::tools::time::epoch_i64();
-
-        if next > now { continue; }
-
-        let mut job = match Job::new(worker_type, &store) {
-            Ok(job) => job,
-            Err(_) => continue, // could not get lock
-        };
-
-        let worker_id = store.clone();
-        let store2 = store.clone();
-        if let Err(err) = WorkerTask::new_thread(
-            worker_type,
-            Some(worker_id),
-            Userid::backup_userid().clone(),
-            false,
-            move |worker| {
-                job.start(&worker.upid().to_string())?;
-                worker.log(format!("starting verification on store {}", store2));
-                worker.log(format!("task triggered by schedule '{}'", event_str));
-                let result = try_block!({
-                    let failed_dirs =
-                        verify_all_backups(datastore, worker.clone(), worker.upid())?;
-                    if failed_dirs.len() > 0 {
-                        worker.log("Failed to verify following snapshots:");
-                        for dir in failed_dirs {
-                            worker.log(format!("\t{}", dir));
-                        }
-                        Err(format_err!("verification failed - please check the log for details"))
-                    } else {
-                        Ok(())
-                    }
-                });
-
-                let status = worker.create_state(&result);
-
-                if let Err(err) = job.finish(status) {
-                    eprintln!("could not finish job state for {}: {}", worker_type, err);
-                }
-
-                result
-            },
-        ) {
-            eprintln!("unable to start verification on store {} - {}", store, err);
-        }
     }
 }
 
 async fn schedule_datastore_sync_jobs() {
 
-    use proxmox_backup::{
-        config::{ sync::{self, SyncJobConfig}, jobstate::{self, Job} },
-        tools::systemd::time::{ parse_calendar_event, compute_next_event },
+    use proxmox_backup::config::sync::{
+        self,
+        SyncJobConfig,
     };
 
     let config = match sync::config() {
@@ -613,94 +476,71 @@ async fn schedule_datastore_sync_jobs() {
             None => continue,
         };
 
-        let event = match parse_calendar_event(&event_str) {
-            Ok(event) => event,
-            Err(err) => {
-                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
-                continue;
-            }
-        };
-
         let worker_type = "syncjob";
+        if check_schedule(worker_type, &event_str, &job_id) {
+            let job = match Job::new(worker_type, &job_id) {
+                Ok(job) => job,
+                Err(_) => continue, // could not get lock
+            };
 
-        let last = match jobstate::last_run_time(worker_type, &job_id) {
-            Ok(time) => time,
-            Err(err) => {
-                eprintln!("could not get last run time of {} {}: {}", worker_type, job_id, err);
-                continue;
+            let auth_id = Authid::root_auth_id().clone();
+            if let Err(err) = do_sync_job(job, job_config, &auth_id, Some(event_str)) {
+                eprintln!("unable to start datastore sync job {} - {}", &job_id, err);
             }
         };
+    }
+}
 
-        let next = match compute_next_event(&event, last, false) {
-            Ok(Some(next)) => next,
-            Ok(None) => continue,
-            Err(err) => {
-                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
-                continue;
-            }
-        };
+async fn schedule_datastore_verify_jobs() {
 
-        let now = proxmox::tools::time::epoch_i64();
+    use proxmox_backup::config::verify::{
+        self,
+        VerificationJobConfig,
+    };
 
-        if next > now  { continue; }
-
-        let job = match Job::new(worker_type, &job_id) {
-            Ok(job) => job,
-            Err(_) => continue, // could not get lock
-        };
-
-        let userid = Userid::backup_userid().clone();
-
-        if let Err(err) = do_sync_job(job, job_config, &userid, Some(event_str)) {
-            eprintln!("unable to start datastore sync job {} - {}", &job_id, err);
+    let config = match verify::config() {
+        Err(err) => {
+            eprintln!("unable to read verification job config - {}", err);
+            return;
         }
+        Ok((config, _digest)) => config,
+    };
+    for (job_id, (_, job_config)) in config.sections {
+        let job_config: VerificationJobConfig = match serde_json::from_value(job_config) {
+            Ok(c) => c,
+            Err(err) => {
+                eprintln!("verification job config from_value failed - {}", err);
+                continue;
+            }
+        };
+        let event_str = match job_config.schedule {
+            Some(ref event_str) => event_str.clone(),
+            None => continue,
+        };
+
+        let worker_type = "verificationjob";
+        let auth_id = Authid::root_auth_id().clone();
+        if check_schedule(worker_type, &event_str, &job_id) {
+            let job = match Job::new(&worker_type, &job_id) {
+                Ok(job) => job,
+                Err(_) => continue, // could not get lock
+            };
+            if let Err(err) = do_verification_job(job, job_config, &auth_id, Some(event_str)) {
+                eprintln!("unable to start datastore verification job {} - {}", &job_id, err);
+            }
+        };
     }
 }
 
 async fn schedule_task_log_rotate() {
-    use proxmox_backup::{
-        config::jobstate::{self, Job},
-        server::rotate_task_log_archive,
-    };
-    use proxmox_backup::server::WorkerTask;
-    use proxmox_backup::tools::systemd::time::{
-        parse_calendar_event, compute_next_event};
 
     let worker_type = "logrotate";
-    let job_id = "task-archive";
-
-    let last = match jobstate::last_run_time(worker_type, job_id) {
-        Ok(time) => time,
-        Err(err) => {
-            eprintln!("could not get last run time of task log archive rotation: {}", err);
-            return;
-        }
-    };
+    let job_id = "access-log_and_task-archive";
 
     // schedule daily at 00:00 like normal logrotate
     let schedule = "00:00";
 
-    let event = match parse_calendar_event(schedule) {
-        Ok(event) => event,
-        Err(err) => {
-            // should not happen?
-            eprintln!("unable to parse schedule '{}' - {}", schedule, err);
-            return;
-        }
-    };
-
-    let next = match compute_next_event(&event, last, false) {
-        Ok(Some(next)) => next,
-        Ok(None) => return,
-        Err(err) => {
-            eprintln!("compute_next_event for '{}' failed - {}", schedule, err);
-            return;
-        }
-    };
-
-    let now = proxmox::tools::time::epoch_i64();
-
-    if next > now {
+    if !check_schedule(worker_type, schedule, job_id) {
         // if we never ran the rotation, schedule instantly
         match jobstate::JobState::load(worker_type, job_id) {
             Ok(state) => match state {
@@ -718,16 +558,16 @@ async fn schedule_task_log_rotate() {
 
     if let Err(err) = WorkerTask::new_thread(
         worker_type,
-        Some(job_id.to_string()),
-        Userid::backup_userid().clone(),
+        None,
+        Authid::root_auth_id().clone(),
         false,
         move |worker| {
             job.start(&worker.upid().to_string())?;
             worker.log(format!("starting task log rotation"));
-            // one entry has normally about ~100-150 bytes
-            let max_size = 500000; // at least 5000 entries
-            let max_files = 20; // at least 100000 entries
+
             let result = try_block!({
+                let max_size = 512 * 1024 - 1; // an entry has ~ 100b, so > 5000 entries/file
+                let max_files = 20; // times twenty files gives > 100000 task entries
                 let has_rotated = rotate_task_log_archive(max_size, true, Some(max_files))?;
                 if has_rotated {
                     worker.log(format!("task log archive was rotated"));
@@ -735,6 +575,28 @@ async fn schedule_task_log_rotate() {
                     worker.log(format!("task log archive was not rotated"));
                 }
 
+                let max_size = 32 * 1024 * 1024 - 1;
+                let max_files = 14;
+                let mut logrotate = LogRotate::new(buildcfg::API_ACCESS_LOG_FN, true)
+                        .ok_or_else(|| format_err!("could not get API access log file names"))?;
+
+                if logrotate.rotate(max_size, None, Some(max_files))? {
+                    println!("rotated access log, telling daemons to re-open log file");
+                    proxmox_backup::tools::runtime::block_on(command_reopen_logfiles())?;
+                    worker.log(format!("API access log was rotated"));
+                } else {
+                    worker.log(format!("API access log was not rotated"));
+                }
+
+                let mut logrotate = LogRotate::new(buildcfg::API_AUTH_LOG_FN, true)
+                        .ok_or_else(|| format_err!("could not get API auth log file names"))?;
+
+                if logrotate.rotate(max_size, None, Some(max_files))? {
+                    worker.log(format!("API access log was rotated"));
+                } else {
+                    worker.log(format!("API access log was not rotated"));
+                }
+
                 Ok(())
             });
 
@@ -752,6 +614,28 @@ async fn schedule_task_log_rotate() {
 
 }
 
+async fn command_reopen_logfiles() -> Result<(), Error> {
+    // only care about the most recent daemon instance for each, proxy & api, as other older ones
+    // should not respond to new requests anyway, but only finish their current one and then exit.
+    let sock = server::our_ctrl_sock();
+    let f1 = server::send_command(sock, serde_json::json!({
+        "command": "api-access-log-reopen",
+    }));
+
+    let pid = server::read_pid(buildcfg::PROXMOX_BACKUP_API_PID_FN)?;
+    let sock = server::ctrl_sock_from_pid(pid);
+    let f2 = server::send_command(sock, serde_json::json!({
+        "command": "api-access-log-reopen",
+    }));
+
+    match futures::join!(f1, f2) {
+        (Err(e1), Err(e2)) => Err(format_err!("reopen commands failed, proxy: {}; api: {}", e1, e2)),
+        (Err(e1), Ok(_)) => Err(format_err!("reopen commands failed, proxy: {}", e1)),
+        (Ok(_), Err(e2)) => Err(format_err!("reopen commands failed, api: {}", e2)),
+        _ => Ok(()),
+    }
+}
+
 async fn run_stat_generator() {
 
     let mut count = 0;
@@ -864,6 +748,36 @@ async fn generate_host_stats(save: bool) {
     });
 }
 
+fn check_schedule(worker_type: &str, event_str: &str, id: &str) -> bool {
+    let event = match parse_calendar_event(event_str) {
+        Ok(event) => event,
+        Err(err) => {
+            eprintln!("unable to parse schedule '{}' - {}", event_str, err);
+            return false;
+        }
+    };
+
+    let last = match jobstate::last_run_time(worker_type, &id) {
+        Ok(time) => time,
+        Err(err) => {
+            eprintln!("could not get last run time of {} {}: {}", worker_type, id, err);
+            return false;
+        }
+    };
+
+    let next = match compute_next_event(&event, last, false) {
+        Ok(Some(next)) => next,
+        Ok(None) => return false,
+        Err(err) => {
+            eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
+            return false;
+        }
+    };
+
+    let now = proxmox::tools::time::epoch_i64();
+    next <= now
+}
+
 fn gather_disk_stats(disk_manager: Arc<DiskManage>, path: &Path, rrd_prefix: &str, save: bool) {
 
     match proxmox_backup::tools::disks::disk_usage(path) {

src/bin/proxmox-daily-update.rs

@@ -0,0 +1,73 @@
+use anyhow::Error;
+use serde_json::{json, Value};
+
+use proxmox::api::{cli::*, RpcEnvironment, ApiHandler};
+
+use proxmox_backup::api2;
+use proxmox_backup::tools::subscription;
+
+async fn wait_for_local_worker(upid_str: &str) -> Result<(), Error> {
+    let upid: proxmox_backup::server::UPID = upid_str.parse()?;
+    let sleep_duration = core::time::Duration::new(0, 100_000_000);
+
+    loop {
+        if !proxmox_backup::server::worker_is_active_local(&upid) {
+            break;
+        }
+        tokio::time::delay_for(sleep_duration).await;
+    }
+    Ok(())
+}
+
+/// Daily update
+async fn do_update(
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Value, Error> {
+    let param = json!({});
+
+    let method = &api2::node::subscription::API_METHOD_CHECK_SUBSCRIPTION;
+    let _res = match method.handler {
+        ApiHandler::Sync(handler) => (handler)(param, method, rpcenv)?,
+        _ => unreachable!(),
+    };
+
+    let notify = match subscription::read_subscription() {
+        Ok(Some(subscription)) => subscription.status == subscription::SubscriptionStatus::ACTIVE,
+        Ok(None) => false,
+        Err(err) => {
+            eprintln!("Error reading subscription - {}", err);
+            false
+        },
+    };
+
+    let param = json!({
+        "notify": notify,
+    });
+    let method = &api2::node::apt::API_METHOD_APT_UPDATE_DATABASE;
+    let upid = match method.handler {
+        ApiHandler::Sync(handler) => (handler)(param, method, rpcenv)?,
+        _ => unreachable!(),
+    };
+    wait_for_local_worker(upid.as_str().unwrap()).await?;
+
+    // TODO: certificate checks/renewal/... ?
+
+    // TODO: cleanup tasks like in PVE?
+
+    Ok(Value::Null)
+}
+
+fn main() {
+    proxmox_backup::tools::setup_safe_path_env();
+
+    let mut rpcenv = CliEnvironment::new();
+    rpcenv.set_auth_id(Some(String::from("root@pam")));
+
+    match proxmox_backup::tools::runtime::main(do_update(&mut rpcenv)) {
+        Err(err) => {
+            eprintln!("error during update: {}", err);
+            std::process::exit(1);
+        },
+        _ => (),
+    }
+}

src/bin/proxmox_backup_client/benchmark.rs

@@ -225,7 +225,7 @@ async fn test_upload_speed(
 
     let backup_time = proxmox::tools::time::epoch_i64();
 
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
     record_repository(&repo);
 
     if verbose { eprintln!("Connecting to backup server"); }

src/bin/proxmox_backup_client/catalog.rs

@@ -79,7 +79,7 @@ async fn dump_catalog(param: Value) -> Result<Value, Error> {
         }
     };
 
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let client = BackupReader::start(
         client,
@@ -153,7 +153,7 @@ async fn dump_catalog(param: Value) -> Result<Value, Error> {
 /// Shell to interactively inspect and restore snapshots.
 async fn catalog_shell(param: Value) -> Result<(), Error> {
     let repo = extract_repository_from_value(&param)?;
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
     let path = tools::required_string_param(&param, "snapshot")?;
     let archive_name = tools::required_string_param(&param, "archive-name")?;
 

src/bin/proxmox_backup_client/key.rs

@@ -414,13 +414,13 @@ fn paperkey_html(data: &str, subject: Option<String>) -> Result<(), Error> {
             println!("</p>");
 
             let data = data.join("\n");
-            let qr_code = generate_qr_code("png", data.as_bytes())?;
+            let qr_code = generate_qr_code("svg", data.as_bytes())?;
             let qr_code = base64::encode_config(&qr_code, base64::STANDARD_NO_PAD);
 
             println!("<center>");
             println!("<img");
             println!("width=\"{}pt\" height=\"{}pt\"", img_size_pt, img_size_pt);
-            println!("src=\"data:image/png;base64,{}\"/>", qr_code);
+            println!("src=\"data:image/svg+xml;base64,{}\"/>", qr_code);
             println!("</center>");
             println!("</div>");
        }
@@ -447,13 +447,13 @@ fn paperkey_html(data: &str, subject: Option<String>) -> Result<(), Error> {
 
     println!("</p>");
 
-    let qr_code = generate_qr_code("png", key_text.as_bytes())?;
+    let qr_code = generate_qr_code("svg", key_text.as_bytes())?;
     let qr_code = base64::encode_config(&qr_code, base64::STANDARD_NO_PAD);
 
     println!("<center>");
     println!("<img");
     println!("width=\"{}pt\" height=\"{}pt\"", img_size_pt, img_size_pt);
-    println!("src=\"data:image/png;base64,{}\"/>", qr_code);
+    println!("src=\"data:image/svg+xml;base64,{}\"/>", qr_code);
     println!("</center>");
 
     println!("</div>");

src/bin/proxmox_backup_client/mount.rs

@@ -144,7 +144,7 @@ fn mount(
     // Process should be deamonized.
     // Make sure to fork before the async runtime is instantiated to avoid troubles.
     let pipe = pipe()?;
-    match fork() {
+    match unsafe { fork() } {
         Ok(ForkResult::Parent { .. }) => {
             nix::unistd::close(pipe.1).unwrap();
             // Blocks the parent process until we are ready to go in the child
@@ -163,7 +163,7 @@ fn mount(
 async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let archive_name = tools::required_string_param(&param, "archive-name")?;
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let target = param["target"].as_str();
 

src/bin/proxmox_backup_client/task.rs

@@ -48,7 +48,7 @@ async fn task_list(param: Value) -> Result<Value, Error> {
     let output_format = get_output_format(&param);
 
     let repo = extract_repository_from_value(&param)?;
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let limit = param["limit"].as_u64().unwrap_or(50) as usize;
     let running = !param["all"].as_bool().unwrap_or(false);
@@ -57,7 +57,7 @@ async fn task_list(param: Value) -> Result<Value, Error> {
         "running": running,
         "start": 0,
         "limit": limit,
-        "userfilter": repo.user(),
+        "userfilter": repo.auth_id(),
         "store": repo.store(),
     });
 
@@ -96,7 +96,7 @@ async fn task_log(param: Value) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let upid =  tools::required_string_param(&param, "upid")?;
 
-    let client = connect(repo.host(), repo.port(), repo.user())?;
+    let client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     display_task_log(client, upid, true).await?;
 
@@ -122,7 +122,7 @@ async fn task_stop(param: Value) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let upid_str =  tools::required_string_param(&param, "upid")?;
 
-    let mut client = connect(repo.host(), repo.port(), repo.user())?;
+    let mut client = connect(repo.host(), repo.port(), repo.auth_id())?;
 
     let path = format!("api2/json/nodes/localhost/tasks/{}", upid_str);
     let _ = client.delete(&path, None).await?;

src/bin/proxmox_backup_manager/acl.rs

@@ -60,7 +60,7 @@ pub fn acl_commands() -> CommandLineInterface {
             "update",
             CliCommand::new(&api2::access::acl::API_METHOD_UPDATE_ACL)
                 .arg_param(&["path", "role"])
-                .completion_cb("userid", config::user::complete_user_name)
+                .completion_cb("auth-id", config::user::complete_authid)
                 .completion_cb("path", config::datastore::complete_acl_path)
 
         );

src/bin/proxmox_backup_manager/mod.rs

@@ -14,5 +14,7 @@ mod sync;
 pub use sync::*;
 mod user;
 pub use user::*;
+mod subscription;
+pub use subscription::*;
 mod disk;
 pub use disk::*;

src/bin/proxmox_backup_manager/subscription.rs

@@ -0,0 +1,55 @@
+use anyhow::Error;
+use serde_json::Value;
+
+use proxmox::api::{api, cli::*, RpcEnvironment, ApiHandler};
+
+use proxmox_backup::api2;
+
+#[api(
+    input: {
+        properties: {
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Read subscription info.
+fn get(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+
+    let output_format = get_output_format(&param);
+
+    let info = &api2::node::subscription::API_METHOD_GET_SUBSCRIPTION;
+    let mut data = match info.handler {
+        ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
+        _ => unreachable!(),
+    };
+
+    let options = default_table_format_options();
+    format_and_print_result_full(&mut data, info.returns, &output_format, &options);
+
+    Ok(Value::Null)
+}
+
+pub fn subscription_commands() -> CommandLineInterface {
+
+    let cmd_def = CliCommandMap::new()
+        .insert("get", CliCommand::new(&API_METHOD_GET))
+        .insert("set",
+            CliCommand::new(&api2::node::subscription::API_METHOD_SET_SUBSCRIPTION)
+                .fixed_param("node", "localhost".into())
+                .arg_param(&["key"])
+        )
+        .insert("update",
+            CliCommand::new(&api2::node::subscription::API_METHOD_CHECK_SUBSCRIPTION)
+                .fixed_param("node", "localhost".into())
+        )
+        .insert("remove",
+            CliCommand::new(&api2::node::subscription::API_METHOD_DELETE_SUBSCRIPTION)
+                .fixed_param("node", "localhost".into())
+        )
+        ;
+
+    cmd_def.into()
+}

src/bin/proxmox_backup_manager/user.rs

@@ -1,11 +1,14 @@
 use anyhow::Error;
 use serde_json::Value;
 
+use std::collections::HashMap;
+
 use proxmox::api::{api, cli::*, RpcEnvironment, ApiHandler};
 
 use proxmox_backup::config;
 use proxmox_backup::tools;
 use proxmox_backup::api2;
+use proxmox_backup::api2::types::{ACL_PATH_SCHEMA, Authid, Userid};
 
 #[api(
     input: {
@@ -48,6 +51,106 @@ fn list_users(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Er
     Ok(Value::Null)
 }
 
+#[api(
+    input: {
+        properties: {
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+            userid: {
+                type: Userid,
+            }
+        }
+    }
+)]
+/// List tokens associated with user.
+fn list_tokens(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+
+    let output_format = get_output_format(&param);
+
+    let info = &api2::access::user::API_METHOD_LIST_TOKENS;
+    let mut data = match info.handler {
+        ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
+        _ => unreachable!(),
+    };
+
+    let options = default_table_format_options()
+        .column(ColumnConfig::new("tokenid"))
+        .column(
+            ColumnConfig::new("enable")
+                .renderer(tools::format::render_bool_with_default_true)
+        )
+        .column(
+            ColumnConfig::new("expire")
+                .renderer(tools::format::render_epoch)
+        )
+        .column(ColumnConfig::new("comment"));
+
+    format_and_print_result_full(&mut data, info.returns, &output_format, &options);
+
+    Ok(Value::Null)
+}
+
+
+#[api(
+    input: {
+        properties: {
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+            "auth-id": {
+                type: Authid,
+            },
+            path: {
+                schema: ACL_PATH_SCHEMA,
+                optional: true,
+            },
+        }
+    }
+)]
+/// List permissions of user/token.
+fn list_permissions(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+
+    let output_format = get_output_format(&param);
+
+    let info = &api2::access::API_METHOD_LIST_PERMISSIONS;
+    let mut data = match info.handler {
+        ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
+        _ => unreachable!(),
+    };
+
+    if output_format == "text" {
+        println!("Privileges with (*) have the propagate flag set\n");
+        let data:HashMap<String, HashMap<String, bool>> = serde_json::from_value(data)?;
+        let mut paths:Vec<String> = data.keys().cloned().collect();
+        paths.sort_unstable();
+        for path in paths {
+            println!("Path: {}", path);
+            let priv_map = data.get(&path).unwrap();
+            let mut privs:Vec<String> = priv_map.keys().cloned().collect();
+            if privs.is_empty() {
+                println!("- NoAccess");
+            } else {
+                privs.sort_unstable();
+                for privilege in privs {
+                    if *priv_map.get(&privilege).unwrap() {
+                        println!("- {} (*)", privilege);
+                    } else {
+                        println!("- {}", privilege);
+                    }
+                }
+            }
+        }
+    } else {
+        format_and_print_result(&mut data, &output_format);
+    }
+
+    Ok(Value::Null)
+}
+
+
 pub fn user_commands() -> CommandLineInterface {
 
     let cmd_def = CliCommandMap::new()
@@ -62,13 +165,39 @@ pub fn user_commands() -> CommandLineInterface {
             "update",
             CliCommand::new(&api2::access::user::API_METHOD_UPDATE_USER)
                 .arg_param(&["userid"])
-                .completion_cb("userid", config::user::complete_user_name)
+                .completion_cb("userid", config::user::complete_userid)
         )
         .insert(
             "remove",
             CliCommand::new(&api2::access::user::API_METHOD_DELETE_USER)
                 .arg_param(&["userid"])
-                .completion_cb("userid", config::user::complete_user_name)
+                .completion_cb("userid", config::user::complete_userid)
+        )
+        .insert(
+            "list-tokens",
+            CliCommand::new(&&API_METHOD_LIST_TOKENS)
+                .arg_param(&["userid"])
+                .completion_cb("userid", config::user::complete_userid)
+        )
+        .insert(
+            "generate-token",
+            CliCommand::new(&api2::access::user::API_METHOD_GENERATE_TOKEN)
+                .arg_param(&["userid", "tokenname"])
+                .completion_cb("userid", config::user::complete_userid)
+        )
+        .insert(
+            "delete-token",
+            CliCommand::new(&api2::access::user::API_METHOD_DELETE_TOKEN)
+                .arg_param(&["userid", "tokenname"])
+                .completion_cb("userid", config::user::complete_userid)
+                .completion_cb("tokenname", config::user::complete_token_name)
+        )
+        .insert(
+            "permissions",
+            CliCommand::new(&&API_METHOD_LIST_PERMISSIONS)
+                .arg_param(&["auth-id"])
+                .completion_cb("auth-id", config::user::complete_authid)
+                .completion_cb("path", config::datastore::complete_acl_path)
         );
 
     cmd_def.into()

src/buildcfg.rs

@@ -4,6 +4,32 @@
 pub const CONFIGDIR: &str = "/etc/proxmox-backup";
 pub const JS_DIR: &str = "/usr/share/javascript/proxmox-backup";
 
+#[macro_export]
+macro_rules! PROXMOX_BACKUP_RUN_DIR_M { () => ("/run/proxmox-backup") }
+
+#[macro_export]
+macro_rules! PROXMOX_BACKUP_LOG_DIR_M { () => ("/var/log/proxmox-backup") }
+
+/// namespaced directory for in-memory (tmpfs) run state
+pub const PROXMOX_BACKUP_RUN_DIR: &str = PROXMOX_BACKUP_RUN_DIR_M!();
+
+/// namespaced directory for persistent logging
+pub const PROXMOX_BACKUP_LOG_DIR: &str = PROXMOX_BACKUP_LOG_DIR_M!();
+
+/// logfile for all API reuests handled by the proxy and privileged API daemons. Note that not all
+/// failed logins can be logged here with full information, use the auth log for that.
+pub const API_ACCESS_LOG_FN: &str = concat!(PROXMOX_BACKUP_LOG_DIR_M!(), "/api/access.log");
+
+/// logfile for any failed authentication, via ticket or via token, and new successfull ticket
+/// creations. This file can be useful for fail2ban.
+pub const API_AUTH_LOG_FN: &str = concat!(PROXMOX_BACKUP_LOG_DIR_M!(), "/api/auth.log");
+
+/// the PID filename for the unprivileged proxy daemon
+pub const PROXMOX_BACKUP_PROXY_PID_FN: &str = concat!(PROXMOX_BACKUP_RUN_DIR_M!(), "/proxy.pid");
+
+/// the PID filename for the privileged api daemon
+pub const PROXMOX_BACKUP_API_PID_FN: &str = concat!(PROXMOX_BACKUP_RUN_DIR_M!(), "/api.pid");
+
 /// Prepend configuration directory to a file name
 ///
 /// This is a simply way to get the full path for configuration files.

src/client/backup_repo.rs

@@ -16,7 +16,7 @@ pub const BACKUP_REPO_URL: ApiStringFormat = ApiStringFormat::Pattern(&BACKUP_RE
 #[derive(Debug)]
 pub struct BackupRepository {
     /// The user name used for Authentication
-    user: Option<Userid>,
+    auth_id: Option<Authid>,
     /// The host name or IP address
     host: Option<String>,
     /// The port
@@ -27,20 +27,29 @@ pub struct BackupRepository {
 
 impl BackupRepository {
 
-    pub fn new(user: Option<Userid>, host: Option<String>, port: Option<u16>, store: String) -> Self {
+    pub fn new(auth_id: Option<Authid>, host: Option<String>, port: Option<u16>, store: String) -> Self {
         let host = match host {
             Some(host) if (IP_V6_REGEX.regex_obj)().is_match(&host) => {
                 Some(format!("[{}]", host))
             },
             other => other,
         };
-        Self { user, host, port, store }
+        Self { auth_id, host, port, store }
+    }
+
+    pub fn auth_id(&self) -> &Authid {
+        if let Some(ref auth_id) = self.auth_id {
+            return auth_id;
+        }
+
+        &Authid::root_auth_id()
     }
 
     pub fn user(&self) -> &Userid {
-        if let Some(ref user) = self.user {
-            return &user;
+        if let Some(auth_id) = &self.auth_id {
+            return auth_id.user();
         }
+
         Userid::root_userid()
     }
 
@@ -65,8 +74,8 @@ impl BackupRepository {
 
 impl fmt::Display for BackupRepository {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match (&self.user, &self.host, self.port) {
-            (Some(user), _, _) => write!(f, "{}@{}:{}:{}", user, self.host(), self.port(), self.store),
+        match (&self.auth_id, &self.host, self.port) {
+            (Some(auth_id), _, _) => write!(f, "{}@{}:{}:{}", auth_id, self.host(), self.port(), self.store),
             (None, Some(host), None) => write!(f, "{}:{}", host, self.store),
             (None, _, Some(port)) => write!(f, "{}:{}:{}", self.host(), port, self.store),
             (None, None, None) => write!(f, "{}", self.store),
@@ -88,7 +97,7 @@ impl std::str::FromStr for BackupRepository {
             .ok_or_else(|| format_err!("unable to parse repository url '{}'", url))?;
 
         Ok(Self {
-            user: cap.get(1).map(|m| Userid::try_from(m.as_str().to_owned())).transpose()?,
+            auth_id: cap.get(1).map(|m| Authid::try_from(m.as_str().to_owned())).transpose()?,
             host: cap.get(2).map(|m| m.as_str().to_owned()),
             port: cap.get(3).map(|m| m.as_str().parse::<u16>()).transpose()?,
             store: cap[4].to_owned(),