bump version to 0.8.13-1

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

.gitignore

@@ -3,3 +3,4 @@ local.mak
 **/*.rs.bk
 /etc/proxmox-backup.service
 /etc/proxmox-backup-proxy.service
+build/

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "proxmox-backup"
-version = "0.2.1"
+version = "0.8.13"
 authors = ["Dietmar Maurer <dietmar@proxmox.com>"]
 edition = "2018"
 license = "AGPL-3"
@@ -14,6 +14,7 @@ name = "proxmox_backup"
 path = "src/lib.rs"
 
 [dependencies]
+apt-pkg-native = "0.3.1" # custom patched version
 base64 = "0.12"
 bitflags = "1.2.1"
 bytes = "0.5"
@@ -30,26 +31,31 @@ lazy_static = "1.4"
 libc = "0.2"
 log = "0.4"
 nix = "0.16"
+num-traits = "0.2"
 once_cell = "1.3.1"
 openssl = "0.10"
 pam = "0.7"
 pam-sys = "0.5"
 percent-encoding = "2.1"
 pin-utils = "0.1.0"
-proxmox = { version = "0.1.38", features = [ "sortable-macro", "api-macro" ] }
+pathpatterns = "0.1.2"
+proxmox = { version = "0.3.3", features = [ "sortable-macro", "api-macro", "websocket" ] }
 #proxmox = { git = "ssh://gitolite3@proxdev.maurer-it.com/rust/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
-#proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro" ] }
+#proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro", "websocket" ] }
+proxmox-fuse = "0.1.0"
+pxar = { version = "0.6.0", features = [ "tokio-io", "futures-io" ] }
+#pxar = { path = "../pxar", features = [ "tokio-io", "futures-io" ] }
 regex = "1.2"
 rustyline = "6"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 siphasher = "0.3"
 syslog = "4.0"
-tokio = { version = "0.2.9", features = [ "blocking", "fs", "io-util", "macros", "rt-threaded", "signal", "stream", "tcp", "time", "uds" ] }
+tokio = { version = "0.2.9", features = [ "blocking", "fs", "dns", "io-util", "macros", "process", "rt-threaded", "signal", "stream", "tcp", "time", "uds" ] }
 tokio-openssl = "0.4.0"
 tokio-util = { version = "0.3", features = [ "codec" ] }
 tower-service = "0.3.0"
-udev = "0.3"
+udev = ">= 0.3, <0.5"
 url = "2.1"
 #valgrind_request = { git = "https://github.com/edef1c/libvalgrind_request", version = "1.1.0", optional = true }
 walkdir = "2"

Makefile

@@ -37,11 +37,15 @@ CARGO ?= cargo
 COMPILED_BINS := \
 	$(addprefix $(COMPILEDIR)/,$(USR_BIN) $(USR_SBIN) $(SERVICE_BIN))
 
+export DEB_VERSION DEB_VERSION_UPSTREAM
+
 SERVER_DEB=${PACKAGE}-server_${DEB_VERSION}_${ARCH}.deb
+SERVER_DBG_DEB=${PACKAGE}-server-dbgsym_${DEB_VERSION}_${ARCH}.deb
 CLIENT_DEB=${PACKAGE}-client_${DEB_VERSION}_${ARCH}.deb
+CLIENT_DBG_DEB=${PACKAGE}-client-dbgsym_${DEB_VERSION}_${ARCH}.deb
 DOC_DEB=${PACKAGE}-docs_${DEB_VERSION}_all.deb
 
-DEBS=${SERVER_DEB} ${CLIENT_DEB}
+DEBS=${SERVER_DEB} ${SERVER_DBG_DEB} ${CLIENT_DEB} ${CLIENT_DBG_DEB}
 
 DSC = rust-${PACKAGE}_${DEB_VERSION}.dsc
 
@@ -65,10 +69,12 @@ doc:
 .PHONY: build
 build:
 	rm -rf build
+	rm -f debian/control
 	debcargo package --config debian/debcargo.toml --changelog-ready --no-overlay-write-back --directory build proxmox-backup $(shell dpkg-parsechangelog -l debian/changelog -SVersion | sed -e 's/-.*//')
 	sed -e '1,/^$$/ ! d' build/debian/control > build/debian/control.src
 	cat build/debian/control.src build/debian/control.in > build/debian/control
 	rm build/debian/control.in build/debian/control.src
+	cp build/debian/control debian/control
 	rm build/Cargo.lock
 	find build/debian -name "*.hint" -delete
 	$(foreach i,$(SUBDIRS), \
@@ -76,18 +82,21 @@ build:
 
 
 .PHONY: proxmox-backup-docs
-proxmox-backup-docs: $(DOC_DEB)
-$(DOC_DEB): build
+$(DOC_DEB) $(DEBS): proxmox-backup-docs
+proxmox-backup-docs: build
 	cd build; dpkg-buildpackage -b -us -uc --no-pre-clean
 	lintian $(DOC_DEB)
 
 # copy the local target/ dir as a build-cache
 .PHONY: deb
-deb: $(DEBS)
-$(DEBS): build
+$(DEBS): deb
+deb: build
 	cd build; dpkg-buildpackage -b -us -uc --no-pre-clean --build-profiles=nodoc
 	lintian $(DEBS)
 
+.PHONY: deb-all
+deb-all: $(DOC_DEB) $(DEBS)
+
 .PHONY: dsc
 dsc: $(DSC)
 $(DSC): build
@@ -140,5 +149,5 @@ install: $(COMPILED_BINS)
 upload: ${SERVER_DEB} ${CLIENT_DEB} ${DOC_DEB}
 	# check if working directory is clean
 	git diff --exit-code --stat && git diff --exit-code --stat --staged
-	tar cf - ${SERVER_DEB} ${DOC_DEB} | ssh -X repoman@repo.proxmox.com upload --product pbs --dist buster
-	tar cf - ${CLIENT_DEB} | ssh -X repoman@repo.proxmox.com upload --product "pbs,pve" --dist buster
+	tar cf - ${SERVER_DEB} ${SERVER_DBG_DEB} ${DOC_DEB} | ssh -X repoman@repo.proxmox.com upload --product pbs --dist buster
+	tar cf - ${CLIENT_DEB} ${CLIENT_DBG_DEB} | ssh -X repoman@repo.proxmox.com upload --product "pbs,pve" --dist buster

TODO.rst

@@ -30,8 +30,6 @@ Chores:
 
 * move tools/xattr.rs and tools/acl.rs to proxmox/sys/linux/
 
-* recompute PXAR_ header types from strings: avoid using numbers from casync
-
 * remove pbs-* systemd timers and services on package purge
 
 

debian/changelog

@@ -1,3 +1,352 @@
+rust-proxmox-backup (0.8.13-1) unstable; urgency=medium
+
+  * improve and add to documentation
+
+  * save last verify result in snapshot manifest and show it in the GUI
+
+  * gc: use human readable units for summary in task log
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 27 Aug 2020 16:12:07 +0200
+
+rust-proxmox-backup (0.8.12-1) unstable; urgency=medium
+
+  * verify: speedup - only verify chunks once
+
+  * verify: sort backup groups
+
+  * bump pxar dep to 0.4.0
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 25 Aug 2020 08:55:52 +0200
+
+rust-proxmox-backup (0.8.11-1) unstable; urgency=medium
+
+  * improve sync jobs, allow to stop them and better logging
+
+  * fix #2926: make network interfaces parser more flexible
+
+  * fix #2904: zpool status: parse also those vdevs without READ/ẀRITE/...
+    statistics
+
+  * api2/node/services: turn service api calls into workers
+
+  * docs: add sections describing ACL related commands and describing
+    benchmarking
+
+  * docs: general grammar, wording and typo improvements
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 19 Aug 2020 19:20:03 +0200
+
+rust-proxmox-backup (0.8.10-1) unstable; urgency=medium
+
+  * ui: acl: add improved permission selector
+
+  * services: make reload safer and default to it in gui
+
+  * ui: rework DataStore content Panel
+
+  * ui: add search box to DataStore content
+
+  * ui: DataStoreContent: keep selection and expansion on reload
+
+  * upload_chunk: allow upload of empty blobs
+
+  * fix #2856: also check whole device for device mapper
+
+  * ui: fix error when reloading DataStoreContent
+
+  * ui: fix in-progress snapshots always showing as "Encrypted"
+
+  * update to pxar 0.3 to support negative timestamps
+
+  * fix #2873: if --pattern is used, default to not extracting
+
+  * finish_backup: test/verify manifest at server side
+
+  * finish_backup: add chunk_upload_stats to manifest
+
+  * src/api2/admin/datastore.rs: add API to get/set Notes for backus
+
+  * list_snapshots: Returns new "comment" property (first line from notes)
+
+  * pxar: create: attempt to use O_NOATIME
+
+  * systemd/time: fix weekday wrapping on month
+
+  * pxar: better error handling on extract
+
+  * pxar/extract: fixup path stack for errors
+
+  * datastore: allow browsing signed pxar files
+
+  * GC: use time pre phase1 to calculate min_atime in phase2
+
+  * gui: user: fix #2898 add dialog to set password
+
+  * fix #2909: handle missing chunks gracefully in garbage collection
+
+  * finish_backup: mark backup as finished only after checks have passed
+
+  * fix: master-key: upload RSA encoded key with backup
+
+  * admin-guide: add section explaining master keys
+
+  * backup: only allow finished backups as base snapshot
+
+  * datastore api: only decode unencrypted indices
+
+  * datastore api: verify blob/index csum from manifest
+
+  * sync, blobs and chunk readers: add more checks and verification
+
+  * verify: add more checks, don't fail on first error
+
+  * mark signed manifests as such
+
+  * backup/prune/forget: improve locking
+
+  * backup: ensure base snapshots are still available after backup
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 11 Aug 2020 15:37:29 +0200
+
+rust-proxmox-backup (0.8.9-1) unstable; urgency=medium
+
+  * improve termprocy (console) behavior on updating proxmox-backup-server and
+    other daemon restarts
+
+  * client: improve upload log output and speed calculation
+
+  * fix #2885: client upload: bail on duplicate backup targets
+
+ -- Proxmox Support Team <support@proxmox.com>  Fri, 24 Jul 2020 11:24:07 +0200
+
+rust-proxmox-backup (0.8.8-1) unstable; urgency=medium
+
+  * pxar: .pxarexclude: match behavior from absolute paths to the one described
+    in the documentation and use byte based paths
+
+  * catalog shell: add exit command
+
+  * manifest: revert signature canonicalization to old behaviour. Fallout from
+    encrypted older backups is expected and was ignored due to the beta status
+    of Proxmox Backup.
+
+  * documentation: various improvements and additions
+
+  * cached user info: print privilege path in error message
+
+  * docs: fix #2851 Add note about GC grace period
+
+  * api2/status: fix datastore full estimation bug if there where (almost) no
+    change for several days
+
+  * schedules, calendar event: support the 'weekly' special expression
+
+  * ui: sync job: group remote fields and use "Source" in labels
+
+  * ui: add calendar event selector
+
+  * ui: sync job: change default to false for "remove-vanished" for new jobs
+
+  * fix #2860: skip in-progress snapshots when syncing
+
+  * fix #2865: detect and skip vanished snapshots
+
+  * fix #2871: close FDs when scanning backup group, avoid leaking
+
+  * backup: list images: handle walkdir error, catch "lost+found" special
+    directory
+
+  * implement AsyncSeek for AsyncIndexReader
+
+  * client: rework logging upload info like size or bandwidth
+
+  * client writer: do not output chunklist for now on verbose=true
+
+  * add initial API for listing available updates and updating the APT
+    database
+
+  * ui: add xterm.js console implementation
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 23 Jul 2020 12:16:05 +0200
+
+rust-proxmox-backup (0.8.7-2) unstable; urgency=medium
+
+  * support restoring file attributes from pxar archives
+
+  * docs: additions and fixes
+
+  * ui: running tasks: update limit to 100
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 14 Jul 2020 12:05:25 +0200
+
+rust-proxmox-backup (0.8.6-1) unstable; urgency=medium
+
+  * ui: add button for easily showing the server fingerprint dashboard
+
+  * proxmox-backup-client benchmark: add --verbose flag and improve output
+    format
+
+  * docs: reference PDF variant in HTML output
+
+  * proxmox-backup-client: add simple version command
+
+  * improve keyfile and signature handling in catalog and manifest
+
+ -- Proxmox Support Team <support@proxmox.com>  Fri, 10 Jul 2020 11:34:14 +0200
+
+rust-proxmox-backup (0.8.5-1) unstable; urgency=medium
+
+  * fix cross process task listing
+
+  * docs: expand datastore documentation
+
+  * docs: add remotes and sync-jobs and schedules
+
+  * bump pathpatterns to 0.1.2
+
+  * ui: align version and user-menu spacing with pve/pmg
+
+  * ui: make username a menu-button
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 09 Jul 2020 15:32:39 +0200
+
+rust-proxmox-backup (0.8.4-1) unstable; urgency=medium
+
+  * add TaskButton in header
+
+  * simpler lost+found pattern
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 09 Jul 2020 14:28:24 +0200
+
+rust-proxmox-backup (0.8.3-1) unstable; urgency=medium
+
+  * get_disks: don't fail on zfs_devices
+
+  * allow some more characters for zpool list
+
+  * ui: adapt for new sign-only crypt mode
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 09 Jul 2020 13:55:06 +0200
+
+rust-proxmox-backup (0.8.2-1) unstable; urgency=medium
+
+  * buildsys: also upload debug packages
+
+  * src/backup/manifest.rs: rename into_string -> to_string
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 09 Jul 2020 11:58:51 +0200
+
+rust-proxmox-backup (0.8.1-1) unstable; urgency=medium
+
+  * remove authhenticated data blobs (not needed)
+
+  * add signature to manifest
+
+  * improve docs
+
+  * client: introduce --keyfd parameter
+
+  * ui improvements
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 09 Jul 2020 10:01:25 +0200
+
+rust-proxmox-backup (0.8.0-1) unstable; urgency=medium
+
+  * implement  get_runtime_with_builder
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 07 Jul 2020 10:15:26 +0200
+
+rust-proxmox-backup (0.7.0-1) unstable; urgency=medium
+
+  *  implement clone for RemoteChunkReader
+
+  * improve docs
+
+  * client: add --encryption boolen parameter
+
+  * client: use default encryption key if it is available
+
+  * d/rules: do not compress .pdf files
+
+  * ui: various fixes
+
+  * add beta text with link to bugtracker
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 07 Jul 2020 07:40:05 +0200
+
+rust-proxmox-backup (0.6.0-1) unstable; urgency=medium
+
+  * make ReadChunk not require mutable self.
+
+  * ui: increase timeout for snapshot listing
+
+  * ui: consistently spell Datastore without space between words
+
+  * ui: disk create: sync and improve 'add-datastore' checkbox label
+
+  * proxmox-backup-client: add benchmark command
+
+  * pxar: fixup 'vanished-file' logic a bit
+
+  * ui: add verify button
+
+ -- Proxmox Support Team <support@proxmox.com>  Fri, 03 Jul 2020 09:45:52 +0200
+
+rust-proxmox-backup (0.5.0-1) unstable; urgency=medium
+
+  * partially revert commit 1f82f9b7b5d231da22a541432d5617cb303c0000
+
+  * ui: allow to Forget (delete) backup snapshots
+
+  * pxar: deal with files changing size during archiving
+
+ -- Proxmox Support Team <support@proxmox.com>  Mon, 29 Jun 2020 13:00:54 +0200
+
+rust-proxmox-backup (0.4.0-1) unstable; urgency=medium
+
+  * change api for incremental backups mode
+
+  * zfs disk management gui
+
+ -- Proxmox Support Team <support@proxmox.com>  Fri, 26 Jun 2020 10:43:27 +0200
+
+rust-proxmox-backup (0.3.0-1) unstable; urgency=medium
+
+  * support incremental backups mode
+
+  * new disk management
+
+  * single file restore for container backups
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 24 Jun 2020 10:12:57 +0200
+
+rust-proxmox-backup (0.2.3-1) unstable; urgency=medium
+
+  * tools/systemd/time: fix compute_next_event for weekdays
+
+  * improve display of 'next run' for sync jobs
+
+  * fix csum calculation for images which do not have a 'chunk_size' aligned
+    size
+
+  * add parser for zpool list output
+
+ -- Proxmox Support Team <support@proxmox.com>  Thu, 04 Jun 2020 10:39:06 +0200
+
+rust-proxmox-backup (0.2.2-1) unstable; urgency=medium
+
+  * proxmox-backup-client.rs: implement quiet flag
+
+  * client restore: don't add server file ending if already specified
+
+  * src/client/pull.rs: also download client.log.blob
+
+  * src/client/pull.rs: more verbose logging
+
+  * gui improvements
+
+ -- Proxmox Support Team <support@proxmox.com>  Wed, 03 Jun 2020 10:37:12 +0200
+
 rust-proxmox-backup (0.2.1-1) unstable; urgency=medium
 
   * ui: move server RRD statistics to 'Server Status' panel

debian/control

@@ -0,0 +1,132 @@
+Source: rust-proxmox-backup
+Section: admin
+Priority: optional
+Build-Depends: debhelper (>= 11),
+ dh-cargo (>= 18),
+ cargo:native,
+ rustc:native,
+ libstd-rust-dev,
+ librust-anyhow-1+default-dev,
+ librust-apt-pkg-native-0.3+default-dev (>= 0.3.1-~~),
+ librust-base64-0.12+default-dev,
+ librust-bitflags-1+default-dev (>= 1.2.1-~~),
+ librust-bytes-0.5+default-dev,
+ librust-chrono-0.4+default-dev,
+ librust-crc32fast-1+default-dev,
+ librust-endian-trait-0.6+arrays-dev,
+ librust-endian-trait-0.6+default-dev,
+ librust-futures-0.3+default-dev,
+ librust-h2-0.2+default-dev,
+ librust-h2-0.2+stream-dev,
+ librust-handlebars-3+default-dev,
+ librust-http-0.2+default-dev,
+ librust-hyper-0.13+default-dev,
+ librust-lazy-static-1+default-dev (>= 1.4-~~),
+ librust-libc-0.2+default-dev,
+ librust-log-0.4+default-dev,
+ librust-nix-0.16+default-dev,
+ librust-nom-5+default-dev (>= 5.1-~~),
+ librust-num-traits-0.2+default-dev,
+ librust-once-cell-1+default-dev (>= 1.3.1-~~),
+ librust-openssl-0.10+default-dev,
+ librust-pam-0.7+default-dev,
+ librust-pam-sys-0.5+default-dev,
+ librust-pathpatterns-0.1+default-dev (>= 0.1.2-~~),
+ librust-percent-encoding-2+default-dev (>= 2.1-~~),
+ librust-pin-utils-0.1+default-dev,
+ librust-proxmox-0.3+api-macro-dev (>= 0.3.3-~~),
+ librust-proxmox-0.3+default-dev (>= 0.3.3-~~),
+ librust-proxmox-0.3+sortable-macro-dev (>= 0.3.3-~~),
+ librust-proxmox-0.3+websocket-dev (>= 0.3.3-~~),
+ librust-proxmox-fuse-0.1+default-dev,
+ librust-pxar-0.6+default-dev,
+ librust-pxar-0.6+futures-io-dev,
+ librust-pxar-0.6+tokio-io-dev,
+ librust-regex-1+default-dev (>= 1.2-~~),
+ librust-rustyline-6+default-dev,
+ librust-serde-1+default-dev,
+ librust-serde-1+derive-dev,
+ librust-serde-json-1+default-dev,
+ librust-siphasher-0.3+default-dev,
+ librust-syslog-4+default-dev,
+ librust-tokio-0.2+blocking-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+default-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+dns-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+fs-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+io-util-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+macros-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+process-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+rt-threaded-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+signal-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+stream-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+tcp-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+time-dev (>= 0.2.9-~~),
+ librust-tokio-0.2+uds-dev (>= 0.2.9-~~),
+ librust-tokio-openssl-0.4+default-dev,
+ librust-tokio-util-0.3+codec-dev,
+ librust-tokio-util-0.3+default-dev,
+ librust-tower-service-0.3+default-dev,
+ librust-udev-0.4+default-dev | librust-udev-0.3+default-dev,
+ librust-url-2+default-dev (>= 2.1-~~),
+ librust-walkdir-2+default-dev,
+ librust-xdg-2+default-dev (>= 2.2-~~),
+ librust-zstd-0.4+bindgen-dev,
+ librust-zstd-0.4+default-dev,
+ libacl1-dev,
+ libfuse3-dev,
+ libsystemd-dev,
+ uuid-dev,
+ debhelper (>= 12~),
+ bash-completion,
+ python3-docutils,
+ python3-pygments,
+ rsync,
+ fonts-dejavu-core <!nodoc>,
+ fonts-lato <!nodoc>,
+ fonts-open-sans <!nodoc>,
+ graphviz <!nodoc>,
+ latexmk <!nodoc>,
+ python3-sphinx <!nodoc>,
+ texlive-fonts-extra <!nodoc>,
+ texlive-fonts-recommended <!nodoc>,
+ texlive-xetex <!nodoc>,
+ xindy <!nodoc>
+Maintainer: Proxmox Support Team <support@proxmox.com>
+Standards-Version: 4.4.1
+Vcs-Git: 
+Vcs-Browser: 
+Homepage: https://www.proxmox.com
+
+Package: proxmox-backup-server
+Architecture: any
+Depends: fonts-font-awesome,
+         libjs-extjs (>= 6.0.1),
+         libzstd1 (>= 1.3.8),
+         lvm2,
+         proxmox-backup-docs,
+         proxmox-mini-journalreader,
+         proxmox-widget-toolkit (>= 2.2-4),
+         pve-xtermjs (>= 4.7.0-1),
+         smartmontools,
+         ${misc:Depends},
+         ${shlibs:Depends},
+Recommends: zfsutils-linux,
+Description: Proxmox Backup Server daemon with tools and GUI
+ This package contains the Proxmox Backup Server daemons and related
+ tools. This includes a web-based graphical user interface.
+
+Package: proxmox-backup-client
+Architecture: any
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Description: Proxmox Backup Client tools
+ This package contains the Proxmox Backup client, which provides a
+ simple command line tool to create and restore backups.
+
+Package: proxmox-backup-docs
+Build-Profiles: <!nodoc>
+Section: doc
+Depends: libjs-extjs,
+         ${misc:Depends},
+Architecture: all
+Description: Proxmox Backup Documentation
+ This package contains the Proxmox Backup Documentation files.

debian/control.in

@@ -3,11 +3,15 @@ Architecture: any
 Depends: fonts-font-awesome,
          libjs-extjs (>= 6.0.1),
          libzstd1 (>= 1.3.8),
+         lvm2,
          proxmox-backup-docs,
          proxmox-mini-journalreader,
          proxmox-widget-toolkit (>= 2.2-4),
+         pve-xtermjs (>= 4.7.0-1),
+         smartmontools,
          ${misc:Depends},
          ${shlibs:Depends},
+Recommends: zfsutils-linux,
 Description: Proxmox Backup Server daemon with tools and GUI
  This package contains the Proxmox Backup Server daemons and related
  tools. This includes a web-based graphical user interface.

debian/lintian-overrides

@@ -0,0 +1,2 @@
+proxmox-backup-server: package-installs-apt-sources etc/apt/sources.list.d/pbstest-beta.list
+proxmox-backup-server: systemd-service-file-refers-to-unusual-wantedby-target lib/systemd/system/proxmox-backup-banner.service getty.target

debian/postinst

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -e
+
+#DEBHELPER#
+
+case "$1" in
+    configure)
+	# modeled after dh_systemd_start output
+	systemctl --system daemon-reload >/dev/null || true
+	if [ -n "$2" ]; then
+		_dh_action=try-reload-or-restart
+	else
+		_dh_action=start
+	fi
+	deb-systemd-invoke $_dh_action proxmox-backup.service proxmox-backup-proxy.service >/dev/null || true
+
+	if test -n "$2"; then
+		if dpkg --compare-versions "$2" 'le' '0.8.10-1'; then
+			echo "Fixing up termproxy user id in task log..."
+			flock -w 30 /var/log/proxmox-backup/tasks/active.lock sed -i 's/:termproxy::root: /:termproxy::root@pam: /' /var/log/proxmox-backup/tasks/active
+		fi
+	fi
+    ;;
+
+    abort-upgrade|abort-remove|abort-deconfigure)
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+exit 0

debian/prerm

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+#DEBHELPER#
+
+# modeled after dh_systemd_start output
+if [ -d /run/systemd/system ] && [ "$1" = remove ]; then
+    deb-systemd-invoke stop 'proxmox-backup-banner.service' 'proxmox-backup-proxy.service' 'proxmox-backup.service' >/dev/null || true
+fi

debian/proxmox-backup-docs.links

@@ -0,0 +1 @@
+/usr/share/doc/proxmox-backup/proxmox-backup.pdf /usr/share/doc/proxmox-backup/html/proxmox-backup.pdf

debian/proxmox-backup-server.install

@@ -1,6 +1,7 @@
 etc/proxmox-backup-proxy.service /lib/systemd/system/
 etc/proxmox-backup.service /lib/systemd/system/
 etc/proxmox-backup-banner.service /lib/systemd/system/
+etc/pbstest-beta.list /etc/apt/sources.list.d/
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-api
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-proxy
 usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-backup-banner

debian/rules

@@ -37,11 +37,14 @@ override_dh_auto_install:
 	    PROXY_USER=backup \
 	    LIBDIR=/usr/lib/$(DEB_HOST_MULTIARCH)
 
-override_dh_installinit:
-	dh_installinit
-	dh_installinit --name proxmox-backup-proxy
+override_dh_installsystemd:
+	# note: we start/try-reload-restart services manually in postinst
+	dh_installsystemd --no-start --no-restart-after-upgrade
 
 # workaround https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=933541
 # TODO: remove once available (Debian 11 ?)
 override_dh_dwz:
 	dh_dwz --no-dwz-multifile
+
+override_dh_compress:
+	dh_compress -X.pdf

docs/Makefile

@@ -1,11 +1,5 @@
 include ../defines.mk
 
-ifeq ($(BUILD_MODE), release)
-COMPILEDIR := ../target/release
-else
-COMPILEDIR := ../target/debug
-endif
-
 GENERATED_SYNOPSIS := 						\
 	proxmox-backup-client/synopsis.rst			\
 	proxmox-backup-client/catalog-shell-synopsis.rst 	\
@@ -26,6 +20,15 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 BUILDDIR      = output
 
+ifeq ($(BUILD_MODE), release)
+COMPILEDIR := ../target/release
+SPHINXOPTS    += -t release
+else
+COMPILEDIR := ../target/debug
+SPHINXOPTS    += -t devbuild
+endif
+
+
 # Sphinx internal variables.
 ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) .
 

docs/administration-guide.rst

@@ -1,9 +1,8 @@
-Administration Guide
-====================
+Backup Management
+=================
 
-The administration guide.
-
-.. todo:: either add a bit more explanation or remove the previous sentence
+.. The administration guide.
+ .. todo:: either add a bit more explanation or remove the previous sentence
 
 Terminology
 -----------
@@ -13,16 +12,16 @@ Backup Content
 
 When doing deduplication, there are different strategies to get
 optimal results in terms of performance and/or deduplication rates.
-Depending on the type of data, one can split data into *fixed* or *variable*
+Depending on the type of data, it can be split into *fixed* or *variable*
 sized chunks.
 
-Fixed sized chunking needs almost no CPU performance, and is used to
+Fixed sized chunking requires minimal CPU power, and is used to
 backup virtual machine images.
 
 Variable sized chunking needs more CPU power, but is essential to get
 good deduplication rates for file archives.
 
-The backup server supports both strategies.
+The Proxmox Backup Server supports both strategies.
 
 
 File Archives: ``<name>.pxar``
@@ -31,7 +30,7 @@ File Archives: ``<name>.pxar``
 .. see https://moinakg.wordpress.com/2013/06/22/high-performance-content-defined-chunking/
 
 A file archive stores a full directory tree. Content is stored using
-the :ref:`pxar-format`, split into variable sized chunks. The format
+the :ref:`pxar-format`, split into variable-sized chunks. The format
 is optimized to achieve good deduplication rates.
 
 
@@ -39,7 +38,7 @@ Image Archives: ``<name>.img``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 This is used for virtual machine images and other large binary
-data. Content is split into fixed sized chunks.
+data. Content is split into fixed-sized chunks.
 
 
 Binary Data (BLOBs)
@@ -56,7 +55,7 @@ Catalog File: ``catalog.pcat1``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The catalog file is an index for file archives. It contains
-the list of files and is used to speed-up search operations.
+the list of files and is used to speed up search operations.
 
 
 The Manifest: ``index.json``
@@ -74,12 +73,12 @@ The backup server groups backups by *type*, where *type* is one of:
 
 ``vm``
     This type is used for :term:`virtual machine`\ s. Typically
-    contains the virtual machine's configuration and an image archive
+    consists of the virtual machine's configuration file and an image archive
     for each disk.
 
 ``ct``
-    This type is used for :term:`container`\ s. Contains the container's
-    configuration and a single file archive for the container content.
+    This type is used for :term:`container`\ s. Consists of the container's
+    configuration and a single file archive for the filesystem content.
 
 ``host``
     This type is used for backups created from within the backed up machine.
@@ -90,7 +89,7 @@ The backup server groups backups by *type*, where *type* is one of:
 Backup ID
 ~~~~~~~~~
 
-An unique ID. Usually the virtual machine or container ID. ``host``
+A unique ID. Usually the virtual machine or container ID. ``host``
 type backups normally use the hostname.
 
 
@@ -122,6 +121,13 @@ uniquely identifies a specific backup within a datastore.
 As you can see, the time format is RFC3399_ with Coordinated
 Universal Time (UTC_, identified by the trailing *Z*).
 
+Backup Server Management
+------------------------
+
+The command line tool to configure and manage the backup server is called
+:command:`proxmox-backup-manager`.
+
+
 
 :term:`DataStore`
 ~~~~~~~~~~~~~~~~~
@@ -133,21 +139,93 @@ or ``zfs``) to store the backup data.
 Datastores are identified by a simple *ID*. You can configure it
 when setting up the backup server.
 
+.. note:: The `File Layout`_ requires the file system to support at least *65538*
+   subdirectories per directory. That number comes from the 2\ :sup:`16`
+   pre-created chunk namespace directories, and the ``.`` and ``..`` default
+   directory entries. This requirement excludes certain filesystems and
+   filesystem configuration from being supported for a datastore. For example,
+   ``ext3`` as a whole or ``ext4`` with the ``dir_nlink`` feature manually disabled.
 
-Backup Server Management
-------------------------
+Disk Management
+~~~~~~~~~~~~~~~
+Proxmox Backup Server comes with a set of disk utilities, which are
+accessed using the ``disk`` subcommand. This subcommand allows you to initialize
+disks, create various filesystems, and get information about the disks.
 
-The command line tool to configure and manage the backup server is called
-:command:`proxmox-backup-manager`.
+To view the disks connected to the system, use the ``list`` subcommand of
+``disk``:
 
+.. code-block:: console
+
+  # proxmox-backup-manager disk list
+  ┌──────┬────────┬─────┬───────────┬─────────────┬───────────────┬─────────┬────────┐
+  │ name │ used   │ gpt │ disk-type │        size │ model         │ wearout │ status │
+  ╞══════╪════════╪═════╪═══════════╪═════════════╪═══════════════╪═════════╪════════╡
+  │ sda  │ lvm    │   1 │ hdd       │ 34359738368 │ QEMU_HARDDISK │       - │ passed │
+  ├──────┼────────┼─────┼───────────┼─────────────┼───────────────┼─────────┼────────┤
+  │ sdb  │ unused │   1 │ hdd       │ 68719476736 │ QEMU_HARDDISK │       - │ passed │
+  ├──────┼────────┼─────┼───────────┼─────────────┼───────────────┼─────────┼────────┤
+  │ sdc  │ unused │   1 │ hdd       │ 68719476736 │ QEMU_HARDDISK │       - │ passed │
+  └──────┴────────┴─────┴───────────┴─────────────┴───────────────┴─────────┴────────┘
+
+To initialize a disk with a new GPT, use the ``initialize`` subcommand:
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk initialize sdX
+
+You can create an ``ext4`` or ``xfs`` filesystem on a disk, using ``fs
+create``. The following command creates an ``ext4`` filesystem and passes the
+``--add-datastore`` parameter, in order to automatically create a datastore on
+the disk (in this case ``sdd``). This will create a datastore at the location
+``/mnt/datastore/store1``:
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk fs create store1 --disk sdd --filesystem ext4 --add-datastore true
+  create datastore 'store1' on disk sdd
+  Percentage done: 1
+  ...
+  Percentage done: 99
+  TASK OK
+
+You can also create a ``zpool`` with various raid levels. The command below
+creates a mirrored ``zpool`` using two disks (``sdb`` & ``sdc``) and mounts it
+on the root directory (default):
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk zpool create zpool1 --devices sdb,sdc --raidlevel mirror
+  create Mirror zpool 'zpool1' on devices 'sdb,sdc'
+  # "zpool" "create" "-o" "ashift=12" "zpool1" "mirror" "sdb" "sdc"
+
+  TASK OK
+
+.. note::
+  You can also pass the ``--add-datastore`` parameter here, to automatically
+  create a datastore from the disk.
+
+You can use ``disk fs list`` and ``disk zpool list`` to keep track of your
+filesystems and zpools respectively.
+
+If a disk supports S.M.A.R.T. capability, and you have this enabled, you can
+display S.M.A.R.T. attributes using the command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager disk smart-attributes sdX
 
 Datastore Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-A :term:`datastore` is a place to store backups. You can configure
-multiple datastores. At least one datastore needs to be
-configured. The datastore is identified by a simple `name` and points
-to a directory.
+You can configure multiple datastores. Minimum one datastore needs to be
+configured. The datastore is identified by a simple `name` and points to a
+directory on the filesystem. Each datastore also has associated retention
+settings of how many backup snapshots for each interval of ``hourly``,
+``daily``, ``weekly``, ``monthly``, ``yearly`` as well as a time-independent
+number of backups to keep in that store. :ref:`Pruning <pruning>` and
+:ref:`garbage collection <garbage-collection>` can also be configured to run
+periodically based on a configured :term:`schedule` per datastore.
 
 The following command creates a new datastore called ``store1`` on :file:`/backup/disk1/store1`
 
@@ -166,6 +244,30 @@ To list existing datastores run:
   │ store1 │ /backup/disk1/store1 │ This is my default storage. │
   └────────┴──────────────────────┴─────────────────────────────┘
 
+You can change settings of a datastore, for example to set a prune and garbage
+collection schedule or retention settings using ``update`` subcommand and view
+a datastore with the ``show`` subcommand:
+
+.. code-block:: console
+
+  # proxmox-backup-manager datastore update store1 --keep-last 7 --prune-schedule daily --gc-schedule 'Tue 04:27'
+  # proxmox-backup-manager datastore show store1
+  ┌────────────────┬─────────────────────────────┐
+  │ Name           │ Value                       │
+  ╞════════════════╪═════════════════════════════╡
+  │ name           │ store1                      │
+  ├────────────────┼─────────────────────────────┤
+  │ path           │ /backup/disk1/store1        │
+  ├────────────────┼─────────────────────────────┤
+  │ comment        │ This is my default storage. │
+  ├────────────────┼─────────────────────────────┤
+  │ gc-schedule    │ Tue 04:27                   │
+  ├────────────────┼─────────────────────────────┤
+  │ keep-last      │ 7                           │
+  ├────────────────┼─────────────────────────────┤
+  │ prune-schedule │ daily                       │
+  └────────────────┴─────────────────────────────┘
+
 Finally, it is possible to remove the datastore configuration:
 
 .. code-block:: console
@@ -179,17 +281,58 @@ Finally, it is possible to remove the datastore configuration:
 File Layout
 ^^^^^^^^^^^
 
-.. todo:: Add datastore file layout example
+After creating a datastore, the following default layout will appear:
+
+.. code-block:: console
+
+  # ls -arilh /backup/disk1/store1
+  276493 -rw-r--r-- 1 backup backup       0 Jul  8 12:35 .lock
+  276490 drwxr-x--- 1 backup backup 1064960 Jul  8 12:35 .chunks
+
+`.lock` is an empty file used for process locking.
+
+The `.chunks` directory contains folders, starting from `0000` and taking hexadecimal values until `ffff`. These
+directories will store the chunked data after a backup operation has been executed.
+
+.. code-block:: console
+
+ # ls -arilh /backup/disk1/store1/.chunks
+ 545824 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 ffff
+ 545823 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fffe
+ 415621 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fffd
+ 415620 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fffc
+ 353187 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fffb
+ 344995 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fffa
+ 144079 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fff9
+ 144078 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fff8
+ 144077 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 fff7
+ ...
+ 403180 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 000c
+ 403179 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 000b
+ 403177 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 000a
+ 402530 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0009
+ 402513 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0008
+ 402509 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0007
+ 276509 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0006
+ 276508 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0005
+ 276507 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0004
+ 276501 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0003
+ 276499 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0002
+ 276498 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0001
+ 276494 drwxr-x--- 2 backup backup 4.0K Jul  8 12:35 0000
+ 276489 drwxr-xr-x 3 backup backup 4.0K Jul  8 12:35 ..
+ 276490 drwxr-x--- 1 backup backup 1.1M Jul  8 12:35 .
+
 
 
 User Management
 ~~~~~~~~~~~~~~~
 
-Proxmox Backup support several authentication realms, and you need to
+Proxmox Backup Server supports several authentication realms, and you need to
 choose the realm when you add a new user. Possible realms are:
 
 :pam: Linux PAM standard authentication. Use this if you want to
-      authenticate as Linux system user (Users needs to exist on the
+      authenticate as Linux system user (Users need to exist on the
       system).
 
 :pbs: Proxmox Backup Server realm. This type stores hashed passwords in
@@ -216,8 +359,8 @@ normally want to add other users with less privileges:
 
   # proxmox-backup-manager user create john@pbs --email john@example.com
 
-The create command lets you specify many option like ``--email`` or
-``--password``, but you can update or change any of them using the
+The create command lets you specify many options like ``--email`` or
+``--password``. You can update or change any of them using the
 update command later:
 
 .. code-block:: console
@@ -225,11 +368,10 @@ update command later:
   # proxmox-backup-manager user update john@pbs --firstname John --lastname Smith
   # proxmox-backup-manager user update john@pbs --comment "An example user."
 
-
 .. todo:: Mention how to set password without passing plaintext password as cli argument.
 
 
-The resulting use list looks like this:
+The resulting user list looks like this:
 
 .. code-block:: console
 
@@ -242,16 +384,16 @@ The resulting use list looks like this:
   │ root@pam │      1 │        │           │          │                  │ Superuser        │
   └──────────┴────────┴────────┴───────────┴──────────┴──────────────────┴──────────────────┘
 
-Newly created users do not have an permissions. Please read the next
+Newly created users do not have any permissions. Please read the next
 section to learn how to set access permissions.
 
-If you want to disable an user account, you can do that by setting ``--enable`` to ``0``
+If you want to disable a user account, you can do that by setting ``--enable`` to ``0``
 
 .. code-block:: console
 
   # proxmox-backup-manager user update john@pbs --enable 0
 
-Or completely remove the users with:
+Or completely remove the user with:
 
 .. code-block:: console
 
@@ -261,20 +403,20 @@ Or completely remove the users with:
 Access Control
 ~~~~~~~~~~~~~~
 
-Users do not have any permission by default. Instead you need to
-specify what is allowed and what not. You can do this by assigning
+By default new users do not have any permission. Instead you need to
+specify what is allowed and what is not. You can do this by assigning
 roles to users on specific objects like datastores or remotes. The
 following roles exist:
 
-**Admin**
-  The Administrator can do anything.
-
-**Audit**
-  An Auditor can view things, but is not allowed to change settings.
-
 **NoAccess**
   Disable Access - nothing is allowed.
 
+**Admin**
+  Can do anything.
+
+**Audit**
+  Can view things, but is not allowed to change settings.
+
 **DatastoreAdmin**
   Can do anything on datastores.
 
@@ -282,10 +424,10 @@ following roles exist:
   Can view datastore settings and list content. But
   is not allowed to read the actual data.
 
-**DataStoreReader**
+**DatastoreReader**
   Can Inspect datastore content and can do restores.
 
-**DataStoreBackup**
+**DatastoreBackup**
   Can backup and restore owned backups.
 
 **DatastorePowerUser**
@@ -300,6 +442,166 @@ following roles exist:
 **RemoteSyncOperator**
   Is allowed to read data from a remote.
 
+You can use the ``acl`` subcommand to manage and monitor user permissions. For
+example, the command below will add the user ``john@pbs`` as a
+**DatastoreAdmin** for the data store ``store1``, located at ``/backup/disk1/store1``:
+
+.. code-block:: console
+
+  # proxmox-backup-manager acl update /datastore/store1 DatastoreAdmin --userid john@pbs
+
+You can monitor the roles of each user using the following command:
+
+.. code-block:: console
+
+   # proxmox-backup-manager acl list
+   ┌──────────┬──────────────────┬───────────┬────────────────┐
+   │ ugid     │ path             │ propagate │ roleid         │
+   ╞══════════╪══════════════════╪═══════════╪════════════════╡
+   │ john@pbs │ /datastore/disk1 │         1 │ DatastoreAdmin │
+   └──────────┴──────────────────┴───────────┴────────────────┘
+
+A single user can be assigned multiple permission sets for different data stores.
+
+.. Note::
+  Naming convention is important here. For data stores on the host,
+  you must use the convention ``/datastore/{storename}``. For example, to set
+  permissions for a data store mounted at ``/mnt/backup/disk4/store2``, you would use
+  ``/datastore/store2`` for the path. For remote stores, use the convention
+  ``/remote/{remote}/{storename}``, where ``{remote}`` signifies the name of the
+  remote (see `Remote` below) and ``{storename}`` is the name of the data store on
+  the remote.
+
+Network Management
+~~~~~~~~~~~~~~~~~~
+Proxmox Backup Server provides an interface for network configuration, through the
+``network`` subcommand. This allows you to carry out some basic network
+management tasks such as adding, configuring and removing network interfaces.
+
+To get a list of available interfaces, use the following command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network list
+  ┌───────┬────────┬───────────┬────────┬─────────┬───────────────────┬──────────────┬──────────────┐
+  │ name  │ type   │ autostart │ method │ method6 │ address           │ gateway      │ ports/slaves │
+  ╞═══════╪════════╪═══════════╪════════╪═════════╪═══════════════════╪══════════════╪══════════════╡
+  │ bond0 │ bond   │         1 │ manual │         │                   │              │ ens18 ens19  │
+  ├───────┼────────┼───────────┼────────┼─────────┼───────────────────┼──────────────┼──────────────┤
+  │ ens18 │ eth    │         1 │ manual │         │                   │              │              │
+  ├───────┼────────┼───────────┼────────┼─────────┼───────────────────┼──────────────┼──────────────┤
+  │ ens19 │ eth    │         1 │ manual │         │                   │              │              │
+  ├───────┼────────┼───────────┼────────┼─────────┼───────────────────┼──────────────┼──────────────┤
+  │ vmbr0 │ bridge │         1 │ static │         │ x.x.x.x/x         │ x.x.x.x      │ bond0        │
+  └───────┴────────┴───────────┴────────┴─────────┴───────────────────┴──────────────┴──────────────┘
+
+To add a new network interface, use the ``create`` subcommand with the relevant
+parameters. The following command shows a template for creating a new bridge:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network create vmbr1 --autostart true --cidr x.x.x.x/x --gateway x.x.x.x --bridge_ports iface_name --type bridge
+
+You can make changes to the configuration of a network interface with the
+``update`` subcommand:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network update vmbr1 --cidr y.y.y.y/y
+
+You can also remove a network interface:
+
+.. code-block:: console
+
+   # proxmox-backup-manager network remove vmbr1
+
+To view the changes made to the network configuration file, before committing
+them, use the command:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network changes
+
+If you would like to cancel all changes at this point, you can do this using:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network revert
+
+If you are happy with the changes and would like to write them into the
+configuration file, the command is:
+
+.. code-block:: console
+
+  # proxmox-backup-manager network reload
+
+You can also configure DNS settings using the ``dns`` subcommand of
+``proxmox-backup-manager``.
+
+:term:`Remote`
+~~~~~~~~~~~~~~
+
+A remote refers to a separate Proxmox Backup Server installation and a user on that
+installation, from which you can `sync` datastores to a local datastore with a
+`Sync Job`.
+
+To add a remote, you need its hostname or ip, a userid and password on the
+remote, and its certificate fingerprint. To get the fingerprint, use the
+``proxmox-backup-manager cert info`` command on the remote.
+
+.. code-block:: console
+
+  # proxmox-backup-manager cert info |grep Fingerprint
+  Fingerprint (sha256): 64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe
+
+Using the information specified above, add the remote with:
+
+.. code-block:: console
+
+  # proxmox-backup-manager remote create pbs2 --host pbs2.mydomain.example --userid sync@pam --password 'SECRET' --fingerprint 64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe
+
+Use the ``list``, ``show``, ``update``, ``remove`` subcommands of
+``proxmox-backup-manager remote`` to manage your remotes:
+
+.. code-block:: console
+
+  # proxmox-backup-manager remote update pbs2 --host pbs2.example
+  # proxmox-backup-manager remote list
+  ┌──────┬──────────────┬──────────┬───────────────────────────────────────────┬─────────┐
+  │ name │ host         │ userid   │ fingerprint                               │ comment │
+  ╞══════╪══════════════╪══════════╪═══════════════════════════════════════════╪═════════╡
+  │ pbs2 │ pbs2.example │ sync@pam │64:d3:ff:3a:50:38:53:5a:9b:f7:50:...:ab:fe │         │
+  └──────┴──────────────┴──────────┴───────────────────────────────────────────┴─────────┘
+  # proxmox-backup-manager remote remove pbs2
+
+
+Sync Jobs
+~~~~~~~~~
+
+Sync jobs are configured to pull the contents of a datastore on a `Remote` to a
+local datastore. You can either start the sync job manually on the GUI or
+provide it with a :term:`schedule` to run regularly. The
+``proxmox-backup-manager sync-job`` command is used to manage sync jobs:
+
+.. code-block:: console
+
+  # proxmox-backup-manager sync-job create pbs2-local --remote pbs2 --remote-store local --store local --schedule 'Wed 02:30'
+  # proxmox-backup-manager sync-job update pbs2-local --comment 'offsite'
+  # proxmox-backup-manager sync-job list
+  ┌────────────┬───────┬────────┬──────────────┬───────────┬─────────┐
+  │ id         │ store │ remote │ remote-store │ schedule  │ comment │
+  ╞════════════╪═══════╪════════╪══════════════╪═══════════╪═════════╡
+  │ pbs2-local │ local │ pbs2   │ local        │ Wed 02:30 │ offsite │
+  └────────────┴───────┴────────┴──────────────┴───────────┴─────────┘
+  # proxmox-backup-manager sync-job remove pbs2-local
+
+Garbage Collection
+~~~~~~~~~~~~~~~~~~
+You can monitor and run :ref:`garbage collection <garbage-collection>` on the
+Proxmox Backup Server using the ``garbage-collection`` subcommand of
+``proxmox-backup-manager``. You can use the ``start`` subcommand to manually start garbage
+collection on an entire data store and the ``status`` subcommand to see
+attributes relating to the :ref:`garbage collection <garbage-collection>`.
 
 
 Backup Client usage
@@ -308,16 +610,16 @@ Backup Client usage
 The command line client is called :command:`proxmox-backup-client`.
 
 
-Respository Locations
-~~~~~~~~~~~~~~~~~~~~~
+Repository Locations
+~~~~~~~~~~~~~~~~~~~~
 
 The client uses the following notation to specify a datastore repository
 on the backup server.
 
   [[username@]server:]datastore
 
-The default value for ``username`` ist ``root``. If no server is specified, the
-default is the local host (``localhost``).
+The default value for ``username`` ist ``root``.  If no server is specified,
+the default is the local host (``localhost``).
 
 You can pass the repository with the ``--repository`` command
 line option, or by setting the ``PBS_REPOSITORY`` environment
@@ -381,7 +683,7 @@ This section explains how to create a backup from within the machine. This can
 be a physical host, a virtual machine, or a container. Such backups may contain file
 and image archives. There are no restrictions in this case.
 
-.. note:: If you want to backup virtual machines or containers on Proxmov VE, see :ref:`pve-integration`.
+.. note:: If you want to backup virtual machines or containers on Proxmox VE, see :ref:`pve-integration`.
 
 For the following example you need to have a backup server set up, working
 credentials and need to know the repository name.
@@ -412,11 +714,13 @@ This will prompt you for a password and then uploads a file archive named
 
 The ``--repository`` option can get quite long and is used by all
 commands. You can avoid having to enter this value by setting the
-environment variable ``PBS_REPOSITORY``.
+environment variable ``PBS_REPOSITORY``. Note that if you would like this to remain set
+over multiple sessions, you should instead add the below line to your
+``.bashrc`` file.
 
 .. code-block:: console
 
-  # export PBS_REPOSTORY=backup-server:store1
+  # export PBS_REPOSITORY=backup-server:store1
 
 After this you can execute all commands without specifying the ``--repository``
 option.
@@ -447,7 +751,7 @@ Excluding files/folders from a backup
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Sometimes it is desired to exclude certain files or folders from a backup archive.
-To tell the Proxmox backup client when and how to ignore files and directories,
+To tell the Proxmox Backup client when and how to ignore files and directories,
 place a text file called ``.pxarexclude`` in the filesystem hierarchy.
 Whenever the backup client encounters such a file in a directory, it interprets
 each line as glob match patterns for files and directories that are to be excluded
@@ -469,17 +773,17 @@ the given patterns. It is only possible to match files in this directory and its
 all files ending in ``.tmp`` within the directory or subdirectories with the
 following pattern ``**/*.tmp``.
 ``[...]`` matches a single character from any of the provided characters within
-the brackets. ``[!...]`` does the complementary and matches any singe character
+the brackets. ``[!...]`` does the complementary and matches any single character
 not contained within the brackets. It is also possible to specify ranges with two
 characters separated by ``-``. For example, ``[a-z]`` matches any lowercase
 alphabetic character and ``[0-9]`` matches any one single digit.
 
-The order of the glob match patterns defines if a file is included or
-excluded, later entries win over previous ones.
+The order of the glob match patterns defines whether a file is included or
+excluded, that is to say later entries override previous ones.
 This is also true for match patterns encountered deeper down the directory tree,
 which can override a previous exclusion.
 Be aware that excluded directories will **not** be read by the backup client.
-A ``.pxarexclude`` file in a subdirectory will have no effect.
+Thus, a ``.pxarexclude`` file in an excluded subdirectory will have no effect.
 ``.pxarexclude`` files are treated as regular files and will be included in the
 backup archive.
 
@@ -529,10 +833,10 @@ Restoring this backup will result in:
     .  ..  file2
 
 Encryption
-^^^^^^^^^^
+~~~~~~~~~~
 
-Proxmox backup supports client side encryption with AES-256 in GCM_
-mode. First you need to create an encryption key:
+Proxmox Backup supports client-side encryption with AES-256 in GCM_
+mode. To set this up, you first need to create an encryption key:
 
 .. code-block:: console
 
@@ -546,6 +850,8 @@ extra protection, you can also create it without a password:
 
   # proxmox-backup-client key create /path/to/my-backup.key --kdf none
 
+Having created this key, it is now possible to create an encrypted backup, by
+passing the ``--keyfile`` parameter, with the path to the key file.
 
 .. code-block:: console
 
@@ -554,23 +860,108 @@ extra protection, you can also create it without a password:
   Encryption Key Password: **************
   ...
 
+.. Note:: If you do not specify the name of the backup key, the key will be
+  created in the default location
+  ``~/.config/proxmox-backup/encryption-key.json``. ``proxmox-backup-client``
+  will also search this location by default, in case the ``--keyfile``
+  parameter is not specified.
 
 You can avoid entering the passwords by setting the environment
 variables ``PBS_PASSWORD`` and ``PBS_ENCRYPTION_PASSWORD``.
 
-.. todo:: Explain master-key
+Using a master key to store and recover encryption keys
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+You can also use ``proxmox-backup-client key`` to create an RSA public/private
+key pair, which can be used to store an encrypted version of the symmetric
+backup encryption key alongside each backup and recover it later.
+
+To set up a master key:
+
+1. Create an encryption key for the backup:
+
+   .. code-block:: console
+
+     # proxmox-backup-client key create
+     creating default key at: "~/.config/proxmox-backup/encryption-key.json"
+     Encryption Key Password: **********
+     ...
+
+   The resulting file will be saved to ``~/.config/proxmox-backup/encryption-key.json``.
+
+2. Create an RSA public/private key pair:
+
+   .. code-block:: console
+
+     # proxmox-backup-client key create-master-key
+     Master Key Password: *********
+     ...
+
+   This will create two files in your current directory, ``master-public.pem``
+   and ``master-private.pem``.
+
+3. Import the newly created ``master-public.pem`` public certificate, so that
+   ``proxmox-backup-client`` can find and use it upon backup.
+
+   .. code-block:: console
+
+     # proxmox-backup-client key import-master-pubkey /path/to/master-public.pem
+     Imported public master key to "~/.config/proxmox-backup/master-public.pem"
+
+4. With all these files in place, run a backup job:
+
+   .. code-block:: console
+
+     # proxmox-backup-client backup etc.pxar:/etc
+
+   The key will be stored in your backup, under the name ``rsa-encrypted.key``.
+
+   .. Note:: The ``--keyfile`` parameter can be excluded, if the encryption key
+     is in the default path. If you specified another path upon creation, you
+     must pass the ``--keyfile`` parameter.
+
+5. To test that everything worked, you can restore the key from the backup:
+
+   .. code-block:: console
+
+     # proxmox-backup-client restore /path/to/backup/ rsa-encrypted.key /path/to/target
+
+   .. Note:: You should not need an encryption key to extract this file. However, if
+     a key exists at the default location
+     (``~/.config/proxmox-backup/encryption-key.json``) the program will prompt
+     you for an encryption key password. Simply moving ``encryption-key.json``
+     out of this directory will fix this issue.
+
+6. Then, use the previously generated master key to decrypt the file:
+
+   .. code-block:: console
+
+     # openssl rsautl -decrypt -inkey master-private.pem -in rsa-encrypted.key -out /path/to/target
+     Enter pass phrase for ./master-private.pem: *********
+
+7. The target file will now contain the encryption key information in plain
+   text. The success of this can be confirmed by passing the resulting ``json``
+   file, with the ``--keyfile`` parameter, when decrypting files from the backup.
+
+.. warning:: Without their key, backed up files will be inaccessible. Thus, you should
+  keep keys ordered and in a place that is separate from the contents being
+  backed up. It can happen, for example, that you back up an entire system, using
+  a key on that system. If the system then becomes inaccessable for any reason
+  and needs to be restored, this will not be possible as the encryption key will be
+  lost along with the broken system. In preparation for the worst case scenario,
+  you should consider keeping a paper copy of this key locked away in
+  a safe place.
 
 Restoring Data
 ~~~~~~~~~~~~~~
 
-The regular creation of backups is a necessary step to avoid data
-loss. More important, however, is the restoration. It is good practice to perform
+The regular creation of backups is a necessary step to avoiding data
+loss. More importantly, however, is the restoration. It is good practice to perform
 periodic recovery tests to ensure that you can access the data in
 case of problems.
 
 First, you need to find the snapshot which you want to restore. The snapshot
-command gives a list of all snapshots on the server:
+command provides a list of all the snapshots on the server:
 
 .. code-block:: console
 
@@ -602,8 +993,8 @@ backup.
 
   # proxmox-backup-client restore host/elsa/2019-12-03T09:35:01Z root.pxar /target/path/
 
-To get the contents of any archive you can restore the ``ìndex.json`` file in the
-repository and restore it to '-'. This will dump the content to the standard output.
+To get the contents of any archive, you can restore the ``index.json`` file in the
+repository to the target path '-'. This will dump the contents to the standard output.
 
 .. code-block:: console
 
@@ -640,13 +1031,13 @@ working directory and list directory contents in the archive.
 ``pwd`` shows the full path of the current working directory with respect to the
 archive root.
 
-Being able to quickly search the contents of the archive is a often needed feature.
+Being able to quickly search the contents of the archive is a commmonly needed feature.
 That's where the catalog is most valuable.
 For example:
 
 .. code-block:: console
 
-  pxar:/ > find etc/ **/*.txt --select
+  pxar:/ > find etc/**/*.txt --select
   "/etc/X11/rgb.txt"
   pxar:/ > list-selected
   etc/**/*.txt
@@ -684,15 +1075,15 @@ file archive as a read-only filesystem to a mountpoint on your host.
 
 .. code-block:: console
 
-  # proxmox-backup-client mount host/backup-client/2020-01-29T11:29:22Z root.pxar /mnt
-  # ls /mnt
+  # proxmox-backup-client mount host/backup-client/2020-01-29T11:29:22Z root.pxar /mnt/mountpoint
+  # ls /mnt/mountpoint
   bin   dev  home  lib32  libx32      media  opt   root  sbin  sys  usr
   boot  etc  lib   lib64  lost+found  mnt    proc  run   srv   tmp  var
 
-This allows you to access the full content of the archive in a seamless manner.
+This allows you to access the full contents of the archive in a seamless manner.
 
 .. note:: As the FUSE connection needs to fetch and decrypt chunks from the
-    backup servers datastore, this can cause some additional network and CPU
+    backup server's datastore, this can cause some additional network and CPU
     load on your host, depending on the operations you perform on the mounted
     filesystem.
 
@@ -700,7 +1091,7 @@ To unmount the filesystem use the ``umount`` command on the mountpoint:
 
 .. code-block:: console
 
-  # umount /mnt
+  # umount /mnt/mountpoint
 
 Login and Logout
 ~~~~~~~~~~~~~~~~
@@ -726,6 +1117,8 @@ To remove the ticket, issue a logout:
   # proxmox-backup-client logout
 
 
+.. _pruning:
+
 Pruning and Removing Backups
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -741,8 +1134,8 @@ command:
    snapshot. They will be inaccessible and unrecoverable.
 
 
-The manual removal is sometimes required, but normally the prune
-command is used to systematically delete older backups. Prune lets
+Although manual removal is sometimes required, the ``prune``
+command is normally used to systematically delete older backups. Prune lets
 you specify which backup snapshots you want to keep. The
 following retention options are available:
 
@@ -787,7 +1180,7 @@ backup is retained.
 
 
 You can use the ``--dry-run`` option to test your settings. This only
-shows the list of existing snapshots and which action prune would take.
+shows the list of existing snapshots and what actions prune would take.
 
 .. code-block:: console
 
@@ -829,6 +1222,17 @@ unused data blocks are removed.
   depending on the number of chunks and the speed of the underlying
   disks.
 
+.. note:: The garbage collection will only remove chunks that haven't been used
+   for at least one day (exactly 24h 5m). This grace period is necessary because
+   chunks in use are marked by touching the chunk which updates the ``atime``
+   (access time) property. Filesystems are mounted with the ``relatime`` option
+   by default. This results in a better performance by only updating the
+   ``atime`` property if the last access has been at least 24 hours ago. The
+   downside is, that touching a chunk within these 24 hours will not always
+   update its ``atime`` property.
+
+   Chunks in the grace period will be logged at the end of the garbage
+   collection task as *Pending removals*.
 
 .. code-block:: console
 
@@ -851,6 +1255,42 @@ unused data blocks are removed.
 
 .. todo:: howto run garbage-collection at regular intervalls (cron)
 
+Benchmarking
+~~~~~~~~~~~~
+The backup client also comes with a benchmarking tool. This tool measures
+various metrics relating to compression and encryption speeds. You can run a
+benchmark using the ``benchmark`` subcommand of ``proxmox-backup-client``:
+
+.. code-block:: console
+
+  # proxmox-backup-client benchmark
+  Uploaded 656 chunks in 5 seconds.
+  Time per request: 7659 microseconds.
+  TLS speed: 547.60 MB/s
+  SHA256 speed: 585.76 MB/s
+  Compression speed: 1923.96 MB/s
+  Decompress speed: 7885.24 MB/s
+  AES256/GCM speed: 3974.03 MB/s
+  ┌───────────────────────────────────┬─────────────────────┐
+  │ Name                              │ Value               │
+  ╞═══════════════════════════════════╪═════════════════════╡
+  │ TLS (maximal backup upload speed) │ 547.60 MB/s (93%)   │
+  ├───────────────────────────────────┼─────────────────────┤
+  │ SHA256 checksum computation speed │ 585.76 MB/s (28%)   │
+  ├───────────────────────────────────┼─────────────────────┤
+  │ ZStd level 1 compression speed    │ 1923.96 MB/s (89%)  │
+  ├───────────────────────────────────┼─────────────────────┤
+  │ ZStd level 1 decompression speed  │ 7885.24 MB/s (98%)  │
+  ├───────────────────────────────────┼─────────────────────┤
+  │ AES256 GCM encryption speed       │ 3974.03 MB/s (104%) │
+  └───────────────────────────────────┴─────────────────────┘
+
+.. note:: The percentages given in the output table correspond to a
+  comparison against a Ryzen 7 2700X. The TLS test connects to the
+  local host, so there is no network involved.
+
+You can also pass the ``--output-format`` parameter to output stats in ``json``,
+rather than the default table format.
 
 .. _pve-integration:
 
@@ -896,7 +1336,3 @@ After that you should be able to see storage status with:
 .. include:: command-line-tools.rst
 
 .. include:: services.rst
-
-.. include host system admin at the end
-
-.. include:: sysadmin.rst

docs/conf.py

@@ -17,7 +17,7 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
+import os
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
@@ -45,8 +45,11 @@ PygmentsBridge.latex_formatter = CustomLatexFormatter
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
+
 extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo"]
 
+todo_link_only = True
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -76,9 +79,11 @@ author = 'Proxmox Support Team'
 # built documents.
 #
 # The short X.Y version.
-version = '0.2'
+vstr = lambda s: '<devbuild>' if s is None else str(s)
+
+version = vstr(os.getenv('DEB_VERSION_UPSTREAM'))
 # The full version, including alpha/beta/rc tags.
-release = '0.2-1'
+release = vstr(os.getenv('DEB_VERSION'))
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -107,7 +112,7 @@ exclude_patterns = [
     'pxar/man1.rst',
     'epilog.rst',
     'pbs-copyright.rst',
-    'sysadmin.rst',
+    'local-zfs.rst'
     'package-repositories.rst',
 ]
 

docs/epilog.rst

@@ -11,8 +11,11 @@
 .. _Container: https://en.wikipedia.org/wiki/Container_(virtualization)
 .. _Zstandard: https://en.wikipedia.org/wiki/Zstandard
 .. _Proxmox: https://www.proxmox.com
+.. _Proxmox Community Forum: https://forum.proxmox.com
 .. _Proxmox Virtual Environment: https://www.proxmox.com/proxmox-ve
-.. _Proxmox Backup: https://www.proxmox.com/proxmox-backup
+  // FIXME
+.. _Proxmox Backup: https://pbs.proxmox.com/wiki/index.php/Main_Page
+.. _PBS Development List: https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
 .. _reStructuredText: https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html
 .. _Rust: https://www.rust-lang.org/
 .. _SHA-256: https://en.wikipedia.org/wiki/SHA-2

docs/glossary.rst

@@ -16,7 +16,7 @@ Glossary
    Datastore
 
       A place to store backups. A directory which contains the backup data.
-      The current implemenation is file-system based.
+      The current implementation is file-system based.
 
    `Rust`_
 
@@ -46,3 +46,19 @@ Glossary
       kernel driver handles filesystem requests and sends them to a
       userspace application.
 
+   Remote
+
+      A remote Proxmox Backup Server installation and credentials for a user on it.
+      You can pull datastores from a remote to a local datastore in order to
+      have redundant backups.
+
+   Schedule
+
+      Certain tasks, for example pruning and garbage collection, need to be
+      performed on a regular basis. Proxmox Backup Server uses a subset of the
+      `systemd Time and Date Specification
+      <https://www.freedesktop.org/software/systemd/man/systemd.time.html#>`_.
+      The subset currently supports time of day specifications and weekdays, in
+      addition to the shorthand expressions 'minutely', 'hourly', 'daily'.
+      There is no support for specifying timezones, the tasks are run in the
+      timezone configured on the server.

docs/index.rst

@@ -1,19 +1,20 @@
 .. Proxmox Backup documentation master file
 
-Welcome to Proxmox Backup's documentation!
-==========================================
+Welcome to the Proxmox Backup documentation!
+============================================
 
-Copyright (C) 2019 Proxmox Server Solutions GmbH
+Copyright (C) 2019-2020 Proxmox Server Solutions GmbH
 
-Permission is granted to copy, distribute and/or modify this document
-under the terms of the GNU Free Documentation License, Version 1.3 or
-any later version published by the Free Software Foundation; with no
-Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A
-copy of the license is included in the section entitled "GNU Free
-Documentation License".
+Permission is granted to copy, distribute and/or modify this document under the
+terms of the GNU Free Documentation License, Version 1.3 or any later version
+published by the Free Software Foundation; with no Invariant Sections, no
+Front-Cover Texts, and no Back-Cover Texts. A copy of the license is included
+in the section entitled "GNU Free Documentation License".
 
-.. todolist::
 
+.. only:: html
+
+ A `PDF` version of the documentation is `also available here <./proxmox-backup.pdf>`_
 
 .. toctree::
    :maxdepth: 3
@@ -22,6 +23,7 @@ Documentation License".
    introduction.rst
    installation.rst
    administration-guide.rst
+   sysadmin.rst
 
 .. raw:: latex
 
@@ -37,5 +39,14 @@ Documentation License".
    glossary.rst
    GFDL.rst
 
+.. only:: html and devbuild
+
+    .. toctree::
+       :maxdepth: 2
+       :caption: Developer Appendix
+
+       todos.rst
+
+
 * :ref:`genindex`
 

docs/installation.rst

@@ -19,9 +19,9 @@ for various management tasks such as disk management.
 The disk image (ISO file) provided by Proxmox includes a complete Debian system
 ("buster" for version 1.x) as well as all necessary packages for the `Proxmox Backup`_ server.
 
-The installer will guide you through the setup process and allows
+The installer will guide you through the setup process and allow
 you to partition the local disk(s), apply basic system configurations
-(e.g. timezone, language, network), and installs all required packages.
+(e.g. timezone, language, network), and install all required packages.
 The provided ISO will get you started in just a few minutes, and is the
 recommended method for new and existing users.
 
@@ -36,11 +36,11 @@ It includes the following:
 
 * The `Proxmox Backup`_ server installer, which partitions the local
   disk(s) with ext4, ext3, xfs or ZFS, and installs the operating
-  system.
+  system
 
 * Complete operating system (Debian Linux, 64-bit)
 
-* Our Linux kernel with ZFS support.
+* Our Linux kernel with ZFS support
 
 * Complete tool-set to administer backups and all necessary resources
 
@@ -54,7 +54,7 @@ Install `Proxmox Backup`_ server on Debian
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Proxmox ships as a set of Debian packages which can be installed on top of a
-standard Debian installation.  After configuring the
+standard Debian installation. After configuring the
 :ref:`sysadmin_package_repositories`, you need to run:
 
 .. code-block:: console
@@ -76,12 +76,15 @@ does, please use the following:
 This will install all required packages, the Proxmox kernel with ZFS_
 support, and a set of common and useful packages.
 
-Installing `Proxmox Backup`_ on top of an existing Debian_ installation looks easy, but
-it presumes that the base system and local storage has been set up correctly.
+.. caution:: Installing `Proxmox Backup`_ on top of an existing Debian_
+  installation looks easy, but it assumes that the base system and local
+  storage have been set up correctly. In general this is not trivial, especially
+  when LVM_ or ZFS_ is used. The network configuration is completely up to you
+  as well.
 
-In general this is not trivial, especially when LVM_ or ZFS_ is used.
-
-The network configuration is completely up to you as well.
+.. note:: You can access the webinterface of the Proxmox Backup Server with
+  your web browser, using HTTPS on port 8007. For example at
+ ``https://<ip-or-dns-name>:8007``
 
 Install Proxmox Backup server on `Proxmox VE`_
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -99,6 +102,10 @@ After configuring the
    server to store backups. Should the hypervisor server fail, you can
    still access the backups.
 
+.. note::
+  You can access the webinterface of the Proxmox Backup Server with your web
+  browser, using HTTPS on port 8007. For example at ``https://<ip-or-dns-name>:8007``
+
 Client installation
 -------------------
 

docs/introduction.rst

@@ -1,120 +1,170 @@
 Introduction
 ============
 
-This documentation is written in :term:`reStructuredText` and formatted with :term:`Sphinx`.
+What is Proxmox Backup Server
+-----------------------------
 
+Proxmox Backup Server is an enterprise-class, client-server backup software
+package that backs up :term:`virtual machine`\ s, :term:`container`\ s, and
+physical hosts. It is specially optimized for the `Proxmox Virtual Environment`_
+platform and allows you to back up your data securely, even between remote
+sites, providing easy management with a web-based user interface.
 
-What is Proxmox Backup
-----------------------
+Proxmox Backup Server supports deduplication, compression, and authenticated
+encryption (AE_). Using :term:`Rust` as the implementation language guarantees high
+performance, low resource usage, and a safe, high-quality codebase.
 
-Proxmox Backup is an enterprise class client-server backup software,
-specially optimized for the `Proxmox Virtual Environment`_ to backup
-:term:`virtual machine`\ s and :term:`container`\ s. It is also
-possible to backup physical hosts.
-
-It supports deduplication, compression and authenticated encryption
-(AE_). Using :term:`Rust` as implementation language guarantees high
-performance, low resource usage, and a safe, high quality code base.
-
-Encryption is done at the client side. This makes backups to not fully
-trusted targets possible.
+It features strong client-side encryption. Thus, it's possible to
+backup data to targets that are not fully trusted.
 
 
 Architecture
 ------------
 
-Proxmox Backup uses a `Client-server model`_. The server is
-responsible to store the backup data and provides an API to create
-backups and restore data. It is possible to manage disks and
-other server side resources using this API.
+Proxmox Backup Server uses a `client-server model`_. The server stores the
+backup data and provides an API to create and manage data stores. With the
+API, it's also possible to manage disks and other server-side resources.
 
-A backup client uses this API to access the backed up data,
-i.e. ``proxmox-backup-client`` is a command line tool to create
-backups and restore data. We deliver an integrated client for
-QEMU_ with `Proxmox Virtual Environment`_.
+The backup client uses this API to access the backed up data. With the command
+line tool ``proxmox-backup-client`` you can create backups and restore data.
+For QEMU_ with `Proxmox Virtual Environment`_ we deliver an integrated client.
 
-A single backup is allowed to contain several archives. For example,
-when you backup a :term:`virtual machine`, each disk is stored as a
-separate archive inside that backup. The VM configuration also gets an
-extra file. This way, it is easy to access and restore important parts
-of the backup without having to scan the whole backup.
+A single backup is allowed to contain several archives. For example, when you
+backup a :term:`virtual machine`, each disk is stored as a separate archive
+inside that backup. The VM configuration itself is stored as an extra file.
+This way, it's easy to access and restore only important parts of the backup,
+without the need to scan the whole backup.
 
 
 Main Features
 -------------
 
-:Proxmox VE: The `Proxmox Virtual Environment`_ is fully
-   supported. You can backup :term:`virtual machine`\ s and
+:Support for Proxmox VE: The `Proxmox Virtual Environment`_ is fully
+   supported and you can easily backup :term:`virtual machine`\ s and
    :term:`container`\ s.
 
-:GUI: We provide a graphical, web based user interface.
+:Performance: The whole software stack is written in :term:`Rust`,
+   in order to provide high speed and memory efficiency.
 
-:Deduplication: Incremental backups produce large amounts of duplicate
-   data. The deduplication layer removes that redundancy and makes
-   incremental backups small and space efficient.
+:Deduplication: Periodic backups produce large amounts of duplicate
+   data. The deduplication layer avoids redundancy and minimizes the storage
+   space used.
 
-:Data Integrity: The built in `SHA-256`_ checksum algorithm assures the
-   accuracy and consistency of your backups.
+:Incremental backups: Changes between backups are typically low. Reading and
+   sending only the delta reduces the storage and network impact of backups.
+
+:Data Integrity: The built-in `SHA-256`_ checksum algorithm ensures accuracy and
+   consistency in your backups.
 
 :Remote Sync: It is possible to efficiently synchronize data to remote
    sites. Only deltas containing new data are transferred.
 
-:Performance: The whole software stack is written in :term:`Rust`,
-   to provide high speed and memory efficiency.
-
-:Compression: Ultra fast Zstandard_ compression is able to compress
+:Compression: The ultra-fast Zstandard_ compression is able to compress
    several gigabytes of data per second.
 
-:Encryption: Backups can be encrypted client-side using AES-256 in
-   GCM_ mode. This authenticated encryption mode (AE_) provides very
-   high performance on modern hardware.
+:Encryption: Backups can be encrypted on the client-side, using AES-256 in
+   Galois/Counter Mode (GCM_) mode. This authenticated encryption (AE_) mode
+   provides very high performance on modern hardware.
 
-:Open Source: No secrets. You have access to all the source code.
+:Web interface: Manage the Proxmox Backup Server with the integrated, web-based
+   user interface.
 
-:Support: Commercial support options are available from `Proxmox`_.
+:Open Source: No secrets. Proxmox Backup Server is free and open-source
+   software. The source code is licensed under AGPL, v3.
+
+:Support: Enterprise support will be available from `Proxmox`_ once the beta
+   phase is over.
 
 
-Why Backup?
------------
+Reasons for Data Backup?
+------------------------
 
-The primary purpose of a backup is to protect against data loss. Data
-loss can be caused by faulty hardware, but also by human error.
+The main purpose of a backup is to protect against data loss. Data loss can be
+caused by both faulty hardware and human error.
 
-A common mistake is to delete a file or folder which is still
-required. Virtualization can amplify this problem. It is now
-easy to delete a whole virtual machine by pressing a single button.
+A common mistake is to accidentally delete a file or folder which is still
+required. Virtualization can even amplify this problem, as deleting a whole
+virtual machine can be as easy as pressing a single button.
 
-Backups can serve as a toolkit for administrators to temporarily
-store data. For example, it is common practice to perform full backups
-before installing major software updates. If something goes wrong, you
-can restore the previous state.
+For administrators, backups can serve as a useful toolkit for temporarily
+storing data. For example, it is common practice to perform full backups before
+installing major software updates. If something goes wrong, you can easily
+restore the previous state.
 
-Another reason for backups are legal requirements. Some data must be
-kept in a safe place for several years by law, so that it can be accessed if
-required.
+Another reason for backups are legal requirements. Some data, especially
+business records, must be kept in a safe place for several years by law, so
+that they can be accessed if required.
 
-Data loss can be very costly as it can severely restrict your
-business. Therefore, make sure that you perform a backup regularly
-and run restore tests.
+In general, data loss is very costly as it can severely damage your business.
+Therefore, ensure that you perform regular backups and run restore tests.
 
 
 Software Stack
 --------------
 
-.. todo:: Eplain why we use Rust (and Flutter)
+Proxmox Backup Server consists of multiple components:
 
+* A server-daemon providing, among other things, a RESTfull API, super-fast
+  asynchronous tasks, lightweight usage statistic collection, scheduling
+  events, strict separation of privileged and unprivileged execution
+  environments
+* A JavaScript management web interface
+* A management CLI tool for the server (`proxmox-backup-manager`)
+* A client CLI tool (`proxmox-backup-client`) to access the server easily from
+  any `Linux amd64` environment
+
+Aside from the web interface, everything is written in the Rust programming
+language.
+
+ "The Rust programming language helps you write faster, more reliable software.
+ High-level ergonomics and low-level control are often at odds in programming
+ language design; Rust challenges that conflict. Through balancing powerful
+ technical capacity and a great developer experience, Rust gives you the option
+ to control low-level details (such as memory usage) without all the hassle
+ traditionally associated with such control."
+
+ -- `The Rust Programming Language <https://doc.rust-lang.org/book/ch00-00-introduction.html>`_
+
+.. todo:: further explain the software stack
+
+Getting Help
+------------
+
+Community Support Forum
+~~~~~~~~~~~~~~~~~~~~~~~
+
+We always encourage our users to discuss and share their knowledge using the
+`Proxmox Community Forum`_. The forum is moderated by the Proxmox support team.
+The large user base is spread out all over the world. Needless to say that such
+a large forum is a great place to get information.
+
+Mailing Lists
+~~~~~~~~~~~~~
+
+Proxmox Backup Server is fully open-source and contributions are welcome! Here
+is the primary communication channel for developers:
+
+:Mailing list for developers: `PBS Development List`_
+
+Bug Tracker
+~~~~~~~~~~~
+
+Proxmox runs a public bug tracker at `<https://bugzilla.proxmox.com>`_. If an
+issue appears, file your report there. An issue can be a bug as well as a
+request for a new feature or enhancement. The bug tracker helps to keep track
+of the issue and will send a notification once it has been solved.
 
 License
 -------
 
-Copyright (C) 2019 Proxmox Server Solutions GmbH
+Copyright (C) 2019-2020 Proxmox Server Solutions GmbH
 
 This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
 
-Proxmox Backup is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Proxmox Backup Server is free and open source software: you can use it,
+redistribute it, and/or modify it under the terms of the GNU Affero General
+Public License as published by the Free Software Foundation, either version 3
+of the License, or (at your option) any later version.
 
 This program is distributed in the hope that it will be useful, but
 ``WITHOUT ANY WARRANTY``; without even the implied warranty of

docs/local-zfs.rst

@@ -0,0 +1,400 @@
+ZFS on Linux
+------------
+
+ZFS is a combined file system and logical volume manager designed by
+Sun Microsystems. There is no need to manually compile ZFS modules - all
+packages are included.
+
+By using ZFS, it's possible to achieve maximum enterprise features with
+low budget hardware, but also high performance systems by leveraging
+SSD caching or even SSD only setups. ZFS can replace cost intense
+hardware raid cards by moderate CPU and memory load combined with easy
+management.
+
+General ZFS advantages
+
+* Easy configuration and management with GUI and CLI.
+* Reliable
+* Protection against data corruption
+* Data compression on file system level
+* Snapshots
+* Copy-on-write clone
+* Various raid levels: RAID0, RAID1, RAID10, RAIDZ-1, RAIDZ-2 and RAIDZ-3
+* Can use SSD for cache
+* Self healing
+* Continuous integrity checking
+* Designed for high storage capacities
+* Asynchronous replication over network
+* Open Source
+* Encryption
+
+Hardware
+~~~~~~~~~
+
+ZFS depends heavily on memory, so you need at least 8GB to start. In
+practice, use as much you can get for your hardware/budget. To prevent
+data corruption, we recommend the use of high quality ECC RAM.
+
+If you use a dedicated cache and/or log disk, you should use an
+enterprise class SSD (e.g. Intel SSD DC S3700 Series). This can
+increase the overall performance significantly.
+
+IMPORTANT: Do not use ZFS on top of hardware controller which has its
+own cache management. ZFS needs to directly communicate with disks. An
+HBA adapter is the way to go, or something like LSI controller flashed
+in ``IT`` mode.
+
+
+ZFS Administration
+~~~~~~~~~~~~~~~~~~
+
+This section gives you some usage examples for common tasks. ZFS
+itself is really powerful and provides many options. The main commands
+to manage ZFS are `zfs` and `zpool`. Both commands come with great
+manual pages, which can be read with:
+
+.. code-block:: console
+
+  # man zpool
+  # man zfs
+
+Create a new zpool
+^^^^^^^^^^^^^^^^^^
+
+To create a new pool, at least one disk is needed. The `ashift` should
+have the same sector-size (2 power of `ashift`) or larger as the
+underlying disk.
+
+.. code-block:: console
+
+  # zpool create -f -o ashift=12 <pool> <device>
+
+Create a new pool with RAID-0
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Minimum 1 disk
+
+.. code-block:: console
+
+  # zpool create -f -o ashift=12 <pool> <device1> <device2>
+
+Create a new pool with RAID-1
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Minimum 2 disks
+
+.. code-block:: console
+
+  # zpool create -f -o ashift=12 <pool> mirror <device1> <device2>
+
+Create a new pool with RAID-10
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Minimum 4 disks
+
+.. code-block:: console
+
+  # zpool create -f -o ashift=12 <pool> mirror <device1> <device2> mirror <device3> <device4>
+
+Create a new pool with RAIDZ-1
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Minimum 3 disks
+
+.. code-block:: console
+
+  # zpool create -f -o ashift=12 <pool> raidz1 <device1> <device2> <device3>
+
+Create a new pool with RAIDZ-2
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Minimum 4 disks
+
+.. code-block:: console
+
+  # zpool create -f -o ashift=12 <pool> raidz2 <device1> <device2> <device3> <device4>
+
+Create a new pool with cache (L2ARC)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It is possible to use a dedicated cache drive partition to increase
+the performance (use SSD).
+
+As `<device>` it is possible to use more devices, like it's shown in
+"Create a new pool with RAID*".
+
+.. code-block:: console
+
+  # zpool create -f -o ashift=12 <pool> <device> cache <cache_device>
+
+Create a new pool with log (ZIL)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It is possible to use a dedicated cache drive partition to increase
+the performance (SSD).
+
+As `<device>` it is possible to use more devices, like it's shown in
+"Create a new pool with RAID*".
+
+.. code-block:: console
+
+  # zpool create -f -o ashift=12 <pool> <device> log <log_device>
+
+Add cache and log to an existing pool
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you have a pool without cache and log. First partition the SSD in
+2 partition with `parted` or `gdisk`
+
+.. important:: Always use GPT partition tables.
+
+The maximum size of a log device should be about half the size of
+physical memory, so this is usually quite small. The rest of the SSD
+can be used as cache.
+
+.. code-block:: console
+
+  # zpool add -f <pool> log <device-part1> cache <device-part2>
+
+
+Changing a failed device
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: console
+
+  # zpool replace -f <pool> <old device> <new device>
+
+
+Changing a failed bootable device
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Depending on how Proxmox Backup was installed it is either using `grub` or `systemd-boot`
+as bootloader.
+
+The first steps of copying the partition table, reissuing GUIDs and replacing
+the ZFS partition are the same. To make the system bootable from the new disk,
+different steps are needed which depend on the bootloader in use.
+
+.. code-block:: console
+
+  # sgdisk <healthy bootable device> -R <new device>
+  # sgdisk -G <new device>
+  # zpool replace -f <pool> <old zfs partition> <new zfs partition>
+
+.. NOTE:: Use the `zpool status -v` command to monitor how far the resilvering process of the new disk has progressed.
+
+With `systemd-boot`:
+
+.. code-block:: console
+
+  # pve-efiboot-tool format <new disk's ESP>
+  # pve-efiboot-tool init <new disk's ESP>
+
+.. NOTE:: `ESP` stands for EFI System Partition, which is setup as partition #2 on
+  bootable disks setup by the {pve} installer since version 5.4. For details, see
+  xref:sysboot_systemd_boot_setup[Setting up a new partition for use as synced ESP].
+
+With `grub`:
+
+Usually `grub.cfg` is located in `/boot/grub/grub.cfg`
+
+.. code-block:: console
+
+  # grub-install <new disk>
+  # grub-mkconfig -o /path/to/grub.cfg
+
+
+Activate E-Mail Notification
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+ZFS comes with an event daemon, which monitors events generated by the
+ZFS kernel module. The daemon can also send emails on ZFS events like
+pool errors. Newer ZFS packages ship the daemon in a separate package,
+and you can install it using `apt-get`:
+
+.. code-block:: console
+
+  # apt-get install zfs-zed
+
+To activate the daemon it is necessary to edit `/etc/zfs/zed.d/zed.rc` with your
+favourite editor, and uncomment the `ZED_EMAIL_ADDR` setting:
+
+.. code-block:: console
+
+  ZED_EMAIL_ADDR="root"
+
+Please note Proxmox Backup forwards mails to `root` to the email address
+configured for the root user.
+
+IMPORTANT: The only setting that is required is `ZED_EMAIL_ADDR`. All
+other settings are optional.
+
+Limit ZFS Memory Usage
+^^^^^^^^^^^^^^^^^^^^^^
+
+It is good to use at most 50 percent (which is the default) of the
+system memory for ZFS ARC to prevent performance shortage of the
+host. Use your preferred editor to change the configuration in
+`/etc/modprobe.d/zfs.conf` and insert:
+
+.. code-block:: console
+
+  options zfs zfs_arc_max=8589934592
+
+This example setting limits the usage to 8GB.
+
+.. IMPORTANT:: If your root file system is ZFS you must update your initramfs every time this value changes:
+
+.. code-block:: console
+
+  # update-initramfs -u
+
+
+SWAP on ZFS
+^^^^^^^^^^^
+
+Swap-space created on a zvol may generate some troubles, like blocking the
+server or generating a high IO load, often seen when starting a Backup
+to an external Storage.
+
+We strongly recommend to use enough memory, so that you normally do not
+run into low memory situations. Should you need or want to add swap, it is
+preferred to create a partition on a physical disk and use it as swapdevice.
+You can leave some space free for this purpose in the advanced options of the
+installer. Additionally, you can lower the `swappiness` value. 
+A good value for servers is 10:
+
+.. code-block:: console
+
+  # sysctl -w vm.swappiness=10
+
+To make the swappiness persistent, open `/etc/sysctl.conf` with
+an editor of your choice and add the following line:
+
+.. code-block:: console
+
+  vm.swappiness = 10
+
+.. table:: Linux kernel `swappiness` parameter values
+  :widths:auto
+
+  ====================  ===============================================================
+   Value                Strategy
+  ====================  ===============================================================
+   vm.swappiness = 0    The kernel will swap only to avoid an 'out of memory' condition
+   vm.swappiness = 1    Minimum amount of swapping without disabling it entirely.
+   vm.swappiness = 10   Sometimes recommended to improve performance when sufficient memory exists in a system.
+   vm.swappiness = 60   The default value.
+   vm.swappiness = 100  The kernel will swap aggressively.
+  ====================  ===============================================================
+
+ZFS Compression
+^^^^^^^^^^^^^^^
+
+To activate compression:
+.. code-block:: console
+
+  # zpool set compression=lz4 <pool>
+
+We recommend using the `lz4` algorithm, since it adds very little CPU overhead.
+Other algorithms such as `lzjb` and `gzip-N` (where `N` is an integer `1-9` representing
+the compression ratio, 1 is fastest and 9 is best compression) are also available.
+Depending on the algorithm and how compressible the data is, having compression enabled can even increase
+I/O performance.
+
+You can disable compression at any time with:
+.. code-block:: console
+
+  # zfs set compression=off <dataset>
+
+Only new blocks will be affected by this change.
+
+ZFS Special Device
+^^^^^^^^^^^^^^^^^^
+
+Since version 0.8.0 ZFS supports `special` devices. A `special` device in a
+pool is used to store metadata, deduplication tables, and optionally small
+file blocks.
+
+A `special` device can improve the speed of a pool consisting of slow spinning
+hard disks with a lot of metadata changes. For example workloads that involve
+creating, updating or deleting a large number of files will benefit from the
+presence of a `special` device. ZFS datasets can also be configured to store
+whole small files on the `special` device which can further improve the
+performance. Use fast SSDs for the `special` device.
+
+.. IMPORTANT:: The redundancy of the `special` device should match the one of the
+  pool, since the `special` device is a point of failure for the whole pool.
+
+.. WARNING:: Adding a `special` device to a pool cannot be undone!
+
+Create a pool with `special` device and RAID-1:
+
+.. code-block:: console
+
+  # zpool create -f -o ashift=12 <pool> mirror <device1> <device2> special mirror <device3> <device4>
+
+Adding a `special` device to an existing pool with RAID-1:
+
+.. code-block:: console
+
+  # zpool add <pool> special mirror <device1> <device2>
+
+ZFS datasets expose the `special_small_blocks=<size>` property. `size` can be
+`0` to disable storing small file blocks on the `special` device or a power of
+two in the range between `512B` to `128K`. After setting the property new file
+blocks smaller than `size` will be allocated on the `special` device.
+
+.. IMPORTANT:: If the value for `special_small_blocks` is greater than or equal to
+  the `recordsize` (default `128K`) of the dataset, *all* data will be written to
+  the `special` device, so be careful!
+
+Setting the `special_small_blocks` property on a pool will change the default
+value of that property for all child ZFS datasets (for example all containers
+in the pool will opt in for small file blocks).
+
+Opt in for all file smaller than 4K-blocks pool-wide:
+
+.. code-block:: console
+
+  # zfs set special_small_blocks=4K <pool>
+
+Opt in for small file blocks for a single dataset:
+
+.. code-block:: console
+
+  # zfs set special_small_blocks=4K <pool>/<filesystem>
+
+Opt out from small file blocks for a single dataset:
+
+.. code-block:: console
+
+  # zfs set special_small_blocks=0 <pool>/<filesystem>
+
+Troubleshooting
+^^^^^^^^^^^^^^^
+
+Corrupted cachefile
+
+In case of a corrupted ZFS cachefile, some volumes may not be mounted during
+boot until mounted manually later.
+
+For each pool, run:
+
+.. code-block:: console
+
+  # zpool set cachefile=/etc/zfs/zpool.cache POOLNAME
+
+and afterwards update the `initramfs` by running:
+
+.. code-block:: console
+
+  # update-initramfs -u -k all
+
+and finally reboot your node.
+
+Sometimes the ZFS cachefile can get corrupted, and `zfs-import-cache.service`
+doesn't import the pools that aren't present in the cachefile.
+
+Another workaround to this problem is enabling the `zfs-import-scan.service`,
+which searches and imports pools via device scanning (usually slower).

docs/package-repositories.rst

@@ -3,17 +3,16 @@
 Debian Package Repositories
 ---------------------------
 
-All Debian based systems use APT_ as package
-management tool. The list of repositories is defined in
-``/etc/apt/sources.list`` and ``.list`` files found in the
-``/etc/apt/sources.d/`` directory. Updates can be installed directly with
-the ``apt`` command line tool, or via the GUI.
+All Debian based systems use APT_ as a package management tool. The lists of
+repositories are defined in ``/etc/apt/sources.list`` and the ``.list`` files found
+in the ``/etc/apt/sources.d/`` directory. Updates can be installed directly
+with the ``apt`` command line tool, or via the GUI.
 
-APT_ ``sources.list`` files list one package repository per line, with
-the most preferred source listed first. Empty lines are ignored and a
-``#`` character anywhere on a line marks the remainder of that line as a
-comment. The information available from the configured sources is
-acquired by ``apt update``.
+APT_ ``sources.list`` files list one package repository per line, with the most
+preferred source listed first. Empty lines are ignored and a ``#`` character
+anywhere on a line marks the remainder of that line as a comment. The
+information available from the configured sources is acquired by ``apt
+update``.
 
 .. code-block:: sources.list
   :caption: File: ``/etc/apt/sources.list``
@@ -27,76 +26,126 @@ acquired by ``apt update``.
 
 .. FIXME for 7.0: change security update suite to bullseye-security
 
-In addition, Proxmox provides three different package repositories for
-the backup server binaries.
+In addition, you need a package repository from Proxmox to get Proxmox Backup updates.
 
-`Proxmox Backup`_ Enterprise Repository
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+During the Proxmox Backup beta phase, only one repository (pbstest) will be
+available. Once released, an Enterprise repository for production use and a
+no-subscription repository will be provided.
 
-This is the default, stable, and recommended repository. It is available for
-all `Proxmox Backup`_ subscription users. It contains the most stable packages,
-and is suitable for production use. The ``pbs-enterprise`` repository is
-enabled by default:
+SecureApt
+~~~~~~~~~
 
-.. code-block:: sources.list
-  :caption: File: ``/etc/apt/sources.list.d/pbs-enterprise.list``
+The `Release` files in the repositories are signed with GnuPG. APT is using
+these signatures to verify that all packages are from a trusted source.
 
-  deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise
+If you install Proxmox Backup Server from an official ISO image, the
+verification key is already installed.
+
+If you install Proxmox Backup Server on top of Debian, download and install the
+key with the following commands:
+
+.. code-block:: console
+
+ # wget http://download.proxmox.com/debian/proxmox-ve-release-6.x.gpg -O /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg
+
+Verify the SHA512 checksum afterwards with:
+
+.. code-block:: console
+
+ # sha512sum /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg
+
+The output should be:
+
+.. code-block:: console
+
+ acca6f416917e8e11490a08a1e2842d500b3a5d9f322c6319db0927b2901c3eae23cfb5cd5df6facf2b57399d3cfa52ad7769ebdd75d9b204549ca147da52626 /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg
+
+and the md5sum:
+
+.. code-block:: console
+
+ # md5sum /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg
+
+Here, the output should be:
+
+.. code-block:: console
+
+ f3f6c5a3a67baf38ad178e5ff1ee270c /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg
+
+.. comment
+    `Proxmox Backup`_ Enterprise Repository
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    This will be the default, stable, and recommended repository. It is available for
+    all `Proxmox Backup`_ subscription users. It contains the most stable packages,
+    and is suitable for production use. The ``pbs-enterprise`` repository is
+    enabled by default:
+
+    .. note:: During the Proxmox Backup beta phase only one repository (pbstest)
+     will be available.
+
+    .. code-block:: sources.list
+      :caption: File: ``/etc/apt/sources.list.d/pbs-enterprise.list``
+
+      deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise
 
 
-To never miss important security fixes, the superuser (``root@pam`` user) is
-notified via email about new packages as soon as they are available. The
-change-log and details of each package can be viewed in the GUI (if available).
+    To never miss important security fixes, the superuser (``root@pam`` user) is
+    notified via email about new packages as soon as they are available. The
+    change-log and details of each package can be viewed in the GUI (if available).
 
-Please note that you need a valid subscription key to access this
-repository. More information regarding subscription levels and pricing can be
-found at https://www.proxmox.com/en/proxmox-backup/pricing.
+    Please note that you need a valid subscription key to access this
+    repository. More information regarding subscription levels and pricing can be
+    found at https://www.proxmox.com/en/proxmox-backup/pricing.
 
-.. note:: You can disable this repository by commenting out the above
-  line using a `#` (at the start of the line). This prevents error
-  messages if you do not have a subscription key. Please configure the
-  ``pbs-no-subscription`` repository in that case.
+    .. note:: You can disable this repository by commenting out the above
+      line using a `#` (at the start of the line). This prevents error
+      messages if you do not have a subscription key. Please configure the
+      ``pbs-no-subscription`` repository in that case.
 
 
-`Proxmox Backup`_ No-Subscription Repository
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    `Proxmox Backup`_ No-Subscription Repository
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-As the name suggests, you do not need a subscription key to access
-this repository. It can be used for testing and non-production
-use. It is not recommended to use it on production servers, because these
-packages are not always heavily tested and validated.
+    As the name suggests, you do not need a subscription key to access
+    this repository. It can be used for testing and non-production
+    use. It is not recommended to use it on production servers, because these
+    packages are not always heavily tested and validated.
 
-We recommend to configure this repository in ``/etc/apt/sources.list``.
+    We recommend to configure this repository in ``/etc/apt/sources.list``.
 
-.. code-block:: sources.list
-  :caption: File: ``/etc/apt/sources.list``
+    .. code-block:: sources.list
+      :caption: File: ``/etc/apt/sources.list``
 
-  deb http://ftp.debian.org/debian buster main contrib
-  deb http://ftp.debian.org/debian buster-updates main contrib
+      deb http://ftp.debian.org/debian buster main contrib
+      deb http://ftp.debian.org/debian buster-updates main contrib
 
-  # PBS pbs-no-subscription repository provided by proxmox.com,
-  # NOT recommended for production use
-  deb http://download.proxmox.com/debian/bps buster pbs-no-subscription
+      # PBS pbs-no-subscription repository provided by proxmox.com,
+      # NOT recommended for production use
+      deb http://download.proxmox.com/debian/pbs buster pbs-no-subscription
 
-  # security updates
-  deb http://security.debian.org/debian-security buster/updates main contrib
+      # security updates
+      deb http://security.debian.org/debian-security buster/updates main contrib
 
 
-`Proxmox Backup`_ Test Repository
+`Proxmox Backup`_ Beta Repository
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Finally, there is a repository called ``pbstest``. This one contains the
-latest packages and is heavily used by developers to test new
+During the public beta, there is a repository called ``pbstest``. This one
+contains the latest packages and is heavily used by developers to test new
 features.
 
-.. warning:: the ``pbstest`` repository should (as the name implies)
+.. .. warning:: the ``pbstest`` repository should (as the name implies)
   only be used to test new features or bug fixes.
 
-You can configure this using ``/etc/apt/sources.list`` by
-adding the following line:
+You can access this repository by adding the following line to
+``/etc/apt/sources.list``:
 
 .. code-block:: sources.list
   :caption: sources.list entry for ``pbstest``
 
-  deb http://download.proxmox.com/debian/bps buster pbstest
+  deb http://download.proxmox.com/debian/pbs buster pbstest
 
+If you installed Proxmox Backup Server from the official beta ISO, you should
+have this repository already configured in
+``/etc/apt/sources.list.d/pbstest-beta.list``

docs/proxmox-backup-proxy/man1.rst

@@ -24,7 +24,7 @@ This daemon is normally started and managed as ``systemd`` service::
 
  systemctl status proxmox-backup-proxy
 
-For debugging, you can start the daemon in forground using::
+For debugging, you can start the daemon in foreground using::
 
  proxmox-backup-proxy
 

docs/pxar/description.rst

@@ -9,7 +9,7 @@ which caters to a similar use-case.
 The ``.pxar`` format is adapted to fulfill the specific needs of the Proxmox
 Backup Server, for example, efficient storage of hardlinks.
 The format is designed to reduce storage space needed on the server by achieving
-a high level of de-duplication.
+a high level of deduplication.
 
 Creating an Archive
 ^^^^^^^^^^^^^^^^^^^
@@ -18,7 +18,7 @@ Run the following command to create an archive of a folder named ``source``:
 
 .. code-block:: console
 
-    # pxar create archive.pxar source
+    # pxar create archive.pxar /path/to/source
 
 This will create a new archive called ``archive.pxar`` with the contents of the
 ``source`` folder.
@@ -29,45 +29,44 @@ This will create a new archive called ``archive.pxar`` with the contents of the
 
 By default, ``pxar`` will skip certain mountpoints and will not follow device
 boundaries. This design decision is based on the primary use case of creating
-archives for backups. It is sensible to not back up the contents of certain
+archives for backups. It makes sense to not back up the contents of certain
 temporary or system specific files.
 To alter this behavior and follow device boundaries, use the
 ``--all-file-systems`` flag.
 
 It is possible to exclude certain files and/or folders from the archive by
-passing glob match patterns as additional parameters. Whenever a file is matched
-by one of the patterns, you will get a warning stating that this file is skipped
-and therefore not included in the archive.
+passing the ``--exclude`` parameter with ``gitignore``\-style match patterns.
 
 For example, you can exclude all files ending in ``.txt`` from the archive
 by running:
 
 .. code-block:: console
 
-    # pxar create archive.pxar source '**/*.txt'
+    # pxar create archive.pxar /path/to/source --exclude '**/*.txt'
 
 Be aware that the shell itself will try to expand all of the glob patterns before
 invoking ``pxar``.
 In order to avoid this, all globs have to be quoted correctly.
  
-It is possible to pass a list of match patterns to fulfill more complex
-file exclusion/inclusion behavior, although it is recommended to use the
+It is possible to pass the ``--exclude`` parameter multiple times, in order to
+match more than one pattern. This allows you to use more complex
+file exclusion/inclusion behavior. However, it is recommended to use
 ``.pxarexclude`` files instead for such cases.
 
 For example you might want to exclude all ``.txt`` files except for a specific
 one from the archive. This is achieved via the negated match pattern, prefixed
 by ``!``.
-All the glob pattern are relative to the ``source`` directory.
+All the glob patterns are relative to the ``source`` directory.
 
 .. code-block:: console
 
-    # pxar create archive.pxar source '**/*.txt' '!/folder/file.txt'
+    # pxar create archive.pxar /path/to/source --exclude '**/*.txt' --exclude '!/folder/file.txt'
 
-.. NOTE:: The order of the glob match patterns matters as later ones win over
+.. NOTE:: The order of the glob match patterns matters as later ones override
     previous ones. Permutations of the same patterns lead to different results.
 
 ``pxar`` will store the list of glob match patterns passed as parameters via the
-command line in a file called ``.pxarexclude-cli`` and stores it at the root of
+command line, in a file called ``.pxarexclude-cli`` at the root of
 the archive.
 If a file with this name is already present in the source folder during archive
 creation, this file is not included in the archive and the file containing the
@@ -86,23 +85,23 @@ The behavior is the same as described in :ref:`creating-backups`.
 Extracting an Archive
 ^^^^^^^^^^^^^^^^^^^^^
 
-An existing archive ``archive.pxar`` is extracted to a ``target`` directory
+An existing archive, ``archive.pxar``, is extracted to a ``target`` directory
 with the following command:
 
 .. code-block:: console
 
-    # pxar extract archive.pxar --target target
+    # pxar extract archive.pxar /path/to/target
 
 If no target is provided, the content of the archive is extracted to the current
 working directory.
 
-In order to restore only parts of an archive, single files and/or folders,
+In order to restore only parts of an archive, single files, and/or folders,
 it is possible to pass the corresponding glob match patterns as additional
-parameters or use the patterns stored in a file:
+parameters or to use the patterns stored in a file:
 
 .. code-block:: console
 
-    # pxar extract etc.pxar '**/*.conf' --target /restore/target/etc
+    # pxar extract etc.pxar /restore/target/etc --pattern '**/*.conf'
 
 The above example restores all ``.conf`` files encountered in any of the
 sub-folders in the archive ``etc.pxar`` to the target ``/restore/target/etc``.

docs/sysadmin.rst

@@ -1,5 +1,5 @@
 Host System Administration
---------------------------
+==========================
 
 `Proxmox Backup`_ is based on the famous Debian_ Linux
 distribution. That means that you have access to the whole world of
@@ -23,8 +23,4 @@ either explain things which are different on `Proxmox Backup`_, or
 tasks which are commonly used on `Proxmox Backup`_. For other topics,
 please refer to the standard Debian documentation.
 
-ZFS
-~~~
-
-.. todo:: Add local ZFS admin guide (local.zfs.adoc)
-
+.. include:: local-zfs.rst

docs/todos.rst

@@ -0,0 +1,6 @@
+Documentation Todo List
+=======================
+
+This is an auto-generated list of the todo references in the documentation.
+
+.. todolist::

etc/Makefile

@@ -7,7 +7,7 @@ DYNAMIC_UNITS := \
 	proxmox-backup.service \
 	proxmox-backup-proxy.service
 
-all: $(UNITS) $(DYNAMIC_UNITS)
+all: $(UNITS) $(DYNAMIC_UNITS) pbstest-beta.list
 
 clean:
 	rm -f $(DYNAMIC_UNITS)

etc/pbstest-beta.list

@@ -0,0 +1 @@
+deb http://download.proxmox.com/debian/pbs buster pbstest

etc/proxmox-backup-proxy.service.in

@@ -2,7 +2,7 @@
 Description=Proxmox Backup API Proxy Server
 Wants=network-online.target
 After=network.target
-Requires=proxmox-backup.service
+Wants=proxmox-backup.service
 After=proxmox-backup.service
 
 [Service]

examples/completion.rs

@@ -49,7 +49,7 @@ fn hello_command(
 }
 
 #[api(input: { properties: {} })]
-/// Quit command. Exit the programm.
+/// Quit command. Exit the program.
 ///
 /// Returns: nothing
 fn quit_command() -> Result<(), Error> {

examples/download-speed.rs

@@ -4,6 +4,7 @@ use anyhow::{Error};
 
 use chrono::{DateTime, Utc};
 
+use proxmox_backup::api2::types::Userid;
 use proxmox_backup::client::{HttpClient, HttpClientOptions, BackupReader};
 
 pub struct DummyWriter {
@@ -27,7 +28,7 @@ async fn run() -> Result<(), Error> {
 
     let host = "localhost";
 
-    let username = "root@pam";
+    let username = Userid::root_userid();
 
     let options = HttpClientOptions::new()
         .interactive(true)
@@ -44,8 +45,8 @@ async fn run() -> Result<(), Error> {
 
     let mut bytes = 0;
     for _ in 0..100 {
-        let writer = DummyWriter { bytes: 0 };
-        let writer = client.speedtest(writer).await?;
+        let mut writer = DummyWriter { bytes: 0 };
+        client.speedtest(&mut writer).await?;
         println!("Received {} bytes", writer.bytes);
         bytes += writer.bytes;
     }
@@ -59,8 +60,7 @@ async fn run() -> Result<(), Error> {
     Ok(())
 }
 
-#[tokio::main]
-async fn main() {
+fn main() {
     if let Err(err) = proxmox_backup::tools::runtime::main(run()) {
         eprintln!("ERROR: {}", err);
     }

examples/dynamic-files.rs

@@ -16,7 +16,7 @@ use std::io::Write;
 // tar: dyntest1/testfile7.dat: File shrank by 2833252864 bytes; padding with zeros
 
 // # pxar create test.pxar ./dyntest1/
-// Error: detected shrinked file "./dyntest1/testfile0.dat" (22020096 < 12679380992)
+// Error: detected shrunk file "./dyntest1/testfile0.dat" (22020096 < 12679380992)
 
 fn create_large_file(path: PathBuf) {
 

examples/upload-speed.rs

@@ -1,13 +1,14 @@
 use anyhow::{Error};
 
+use proxmox_backup::api2::types::Userid;
 use proxmox_backup::client::*;
 
-async fn upload_speed() -> Result<usize, Error> {
+async fn upload_speed() -> Result<f64, Error> {
 
     let host = "localhost";
     let datastore = "store2";
 
-    let username = "root@pam";
+    let username = Userid::root_userid();
 
     let options = HttpClientOptions::new()
         .interactive(true)
@@ -17,10 +18,10 @@ async fn upload_speed() -> Result<usize, Error> {
 
     let backup_time = chrono::Utc::now();
 
-    let client = BackupWriter::start(client, datastore, "host", "speedtest", backup_time, false).await?;
+    let client = BackupWriter::start(client, None, datastore, "host", "speedtest", backup_time, false).await?;
 
     println!("start upload speed test");
-    let res = client.upload_speedtest().await?;
+    let res = client.upload_speedtest(true).await?;
 
     Ok(res)
 }

src/api2.rs

@@ -4,10 +4,11 @@ pub mod backup;
 pub mod config;
 pub mod node;
 pub mod reader;
-mod subscription;
+pub mod status;
 pub mod types;
 pub mod version;
 pub mod pull;
+mod helpers;
 
 use proxmox::api::router::SubdirMap;
 use proxmox::api::Router;
@@ -23,7 +24,7 @@ pub const SUBDIRS: SubdirMap = &[
     ("nodes", &NODES_ROUTER),
     ("pull", &pull::ROUTER),
     ("reader", &reader::ROUTER),
-    ("subscription", &subscription::ROUTER),
+    ("status", &status::ROUTER),
     ("version", &version::ROUTER),
 ];
 

src/api2/access.rs

@@ -2,56 +2,110 @@ use anyhow::{bail, format_err, Error};
 
 use serde_json::{json, Value};
 
-use proxmox::api::{api, RpcEnvironment, Permission, UserInformation};
+use proxmox::api::{api, RpcEnvironment, Permission};
 use proxmox::api::router::{Router, SubdirMap};
 use proxmox::{sortable, identity};
 use proxmox::{http_err, list_subdirs_api_method};
 
-use crate::tools;
-use crate::tools::ticket::*;
+use crate::tools::ticket::{self, Empty, Ticket};
 use crate::auth_helpers::*;
 use crate::api2::types::*;
 
 use crate::config::cached_user_info::CachedUserInfo;
-use crate::config::acl::PRIV_PERMISSIONS_MODIFY;
+use crate::config::acl::{PRIVILEGES, PRIV_PERMISSIONS_MODIFY};
 
 pub mod user;
 pub mod domain;
 pub mod acl;
 pub mod role;
 
-fn authenticate_user(username: &str, password: &str) -> Result<(), Error> {
-
+/// returns Ok(true) if a ticket has to be created
+/// and Ok(false) if not
+fn authenticate_user(
+    userid: &Userid,
+    password: &str,
+    path: Option<String>,
+    privs: Option<String>,
+    port: Option<u16>,
+) -> Result<bool, Error> {
     let user_info = CachedUserInfo::new()?;
 
-    if !user_info.is_active_user(&username) {
+    if !user_info.is_active_user(&userid) {
         bail!("user account disabled or expired.");
     }
 
-    let ticket_lifetime = tools::ticket::TICKET_LIFETIME;
-
     if password.starts_with("PBS:") {
-        if let Ok((_age, Some(ticket_username))) = tools::ticket::verify_rsa_ticket(public_auth_key(), "PBS", password, None, -300, ticket_lifetime) {
-            if ticket_username == username {
-                return Ok(());
-            } else {
-                bail!("ticket login failed - wrong username");
+        if let Ok(ticket_userid) = Ticket::<Userid>::parse(password)
+            .and_then(|ticket| ticket.verify(public_auth_key(), "PBS", None))
+        {
+            if *userid == ticket_userid {
+                return Ok(true);
             }
+            bail!("ticket login failed - wrong userid");
+        }
+    } else if password.starts_with("PBSTERM:") {
+        if path.is_none() || privs.is_none() || port.is_none() {
+            bail!("cannot check termnal ticket without path, priv and port");
+        }
+
+        let path = path.ok_or_else(|| format_err!("missing path for termproxy ticket"))?;
+        let privilege_name = privs
+            .ok_or_else(|| format_err!("missing privilege name for termproxy ticket"))?;
+        let port = port.ok_or_else(|| format_err!("missing port for termproxy ticket"))?;
+
+        if let Ok(Empty) = Ticket::parse(password)
+            .and_then(|ticket| ticket.verify(
+                public_auth_key(),
+                ticket::TERM_PREFIX,
+                Some(&ticket::term_aad(userid, &path, port)),
+            ))
+        {
+            for (name, privilege) in PRIVILEGES {
+                if *name == privilege_name {
+                    let mut path_vec = Vec::new();
+                    for part in path.split('/') {
+                        if part != "" {
+                            path_vec.push(part);
+                        }
+                    }
+
+                    user_info.check_privs(userid, &path_vec, *privilege, false)?;
+                    return Ok(false);
+                }
+            }
+
+            bail!("No such privilege");
         }
     }
 
-    crate::auth::authenticate_user(username, password)
+    let _ = crate::auth::authenticate_user(userid, password)?;
+    Ok(true)
 }
 
 #[api(
     input: {
         properties: {
             username: {
-                schema: PROXMOX_USER_ID_SCHEMA,
+                type: Userid,
             },
             password: {
                 schema: PASSWORD_SCHEMA,
             },
+            path: {
+                type: String,
+                description: "Path for verifying terminal tickets.",
+                optional: true,
+            },
+            privs: {
+                type: String,
+                description: "Privilege for verifying terminal tickets.",
+                optional: true,
+            },
+            port: {
+                type: Integer,
+                description: "Port for verifying terminal tickets.",
+                optional: true,
+            },
         },
     },
     returns: {
@@ -78,11 +132,16 @@ fn authenticate_user(username: &str, password: &str) -> Result<(), Error> {
 /// Create or verify authentication ticket.
 ///
 /// Returns: An authentication ticket with additional infos.
-fn create_ticket(username: String, password: String) -> Result<Value, Error> {
-    match authenticate_user(&username, &password) {
-        Ok(_) => {
-
-            let ticket = assemble_rsa_ticket( private_auth_key(), "PBS", Some(&username), None)?;
+fn create_ticket(
+    username: Userid,
+    password: String,
+    path: Option<String>,
+    privs: Option<String>,
+    port: Option<u16>,
+) -> Result<Value, Error> {
+    match authenticate_user(&username, &password, path, privs, port) {
+        Ok(true) => {
+            let ticket = Ticket::new("PBS", &username)?.sign(private_auth_key(), None)?;
 
             let token = assemble_csrf_prevention_token(csrf_secret(), &username);
 
@@ -94,10 +153,13 @@ fn create_ticket(username: String, password: String) -> Result<Value, Error> {
                 "CSRFPreventionToken": token,
             }))
         }
+        Ok(false) => Ok(json!({
+            "username": username,
+        })),
         Err(err) => {
             let client_ip = "unknown"; // $rpcenv->get_client_ip() || '';
             log::error!("authentication failure; rhost={} user={} msg={}", client_ip, username, err.to_string());
-            Err(http_err!(UNAUTHORIZED, "permission check failed.".into()))
+            Err(http_err!(UNAUTHORIZED, "permission check failed."))
         }
     }
 }
@@ -106,7 +168,7 @@ fn create_ticket(username: String, password: String) -> Result<Value, Error> {
     input: {
         properties: {
             userid: {
-                schema: PROXMOX_USER_ID_SCHEMA,
+                type: Userid,
             },
             password: {
                 schema: PASSWORD_SCHEMA,
@@ -124,13 +186,15 @@ fn create_ticket(username: String, password: String) -> Result<Value, Error> {
 /// Each user is allowed to change his own password. Superuser
 /// can change all passwords.
 fn change_password(
-    userid: String,
+    userid: Userid,
     password: String,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
 
-    let current_user = rpcenv.get_user()
-        .ok_or_else(|| format_err!("unknown user"))?;
+    let current_user: Userid = rpcenv
+        .get_user()
+        .ok_or_else(|| format_err!("unknown user"))?
+        .parse()?;
 
     let mut allowed = userid == current_user;
 
@@ -146,9 +210,8 @@ fn change_password(
         bail!("you are not authorized to change the password.");
     }
 
-    let (username, realm) = crate::auth::parse_userid(&userid)?;
-    let authenticator = crate::auth::lookup_authenticator(&realm)?;
-    authenticator.store_password(&username, &password)?;
+    let authenticator = crate::auth::lookup_authenticator(userid.realm())?;
+    authenticator.store_password(userid.name(), &password)?;
 
     Ok(Value::Null)
 }

src/api2/access/acl.rs

@@ -2,6 +2,7 @@ use anyhow::{bail, Error};
 use ::serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, Router, RpcEnvironment, Permission};
+use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
 use crate::config::acl;
@@ -141,7 +142,7 @@ pub fn read_acl(
             },
             userid: {
                 optional: true,
-                schema: PROXMOX_USER_ID_SCHEMA,
+                type: Userid,
             },
             group: {
                 optional: true,
@@ -167,14 +168,14 @@ pub fn update_acl(
     path: String,
     role: String,
     propagate: Option<bool>,
-    userid: Option<String>,
+    userid: Option<Userid>,
     group: Option<String>,
     delete: Option<bool>,
     digest: Option<String>,
     _rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(acl::ACL_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(acl::ACL_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let (mut tree, expected_digest) = acl::config()?;
 
@@ -192,7 +193,7 @@ pub fn update_acl(
     } else if let Some(ref userid) = userid {
         if !delete { // Note: we allow to delete non-existent users
             let user_cfg = crate::config::user::cached_config()?;
-            if user_cfg.sections.get(userid).is_none() {
+            if user_cfg.sections.get(&userid.to_string()).is_none() {
                 bail!("no such user.");
             }
         }

src/api2/access/user.rs

@@ -3,6 +3,7 @@ use serde_json::Value;
 
 use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
 use proxmox::api::schema::{Schema, StringSchema};
+use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
 use crate::config::user;
@@ -48,7 +49,7 @@ pub fn list_users(
     input: {
         properties: {
             userid: {
-                schema: PROXMOX_USER_ID_SCHEMA,
+                type: Userid,
             },
             comment: {
                 schema: SINGLE_LINE_COMMENT_SCHEMA,
@@ -87,25 +88,24 @@ pub fn list_users(
 /// Create new user.
 pub fn create_user(password: Option<String>, param: Value) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let user: user::User = serde_json::from_value(param)?;
 
     let (mut config, _digest) = user::config()?;
 
-    if let Some(_) = config.sections.get(&user.userid) {
+    if let Some(_) = config.sections.get(user.userid.as_str()) {
         bail!("user '{}' already exists.", user.userid);
     }
 
-    let (username, realm) = crate::auth::parse_userid(&user.userid)?;
-    let authenticator = crate::auth::lookup_authenticator(&realm)?;
+    let authenticator = crate::auth::lookup_authenticator(&user.userid.realm())?;
 
-    config.set_data(&user.userid, "user", &user)?;
+    config.set_data(user.userid.as_str(), "user", &user)?;
 
     user::save_config(&config)?;
 
     if let Some(password) = password {
-        authenticator.store_password(&username, &password)?;
+        authenticator.store_password(user.userid.name(), &password)?;
     }
 
     Ok(())
@@ -115,7 +115,7 @@ pub fn create_user(password: Option<String>, param: Value) -> Result<(), Error>
    input: {
         properties: {
             userid: {
-                schema: PROXMOX_USER_ID_SCHEMA,
+                type: Userid,
             },
          },
     },
@@ -128,9 +128,9 @@ pub fn create_user(password: Option<String>, param: Value) -> Result<(), Error>
     },
 )]
 /// Read user configuration data.
-pub fn read_user(userid: String, mut rpcenv: &mut dyn RpcEnvironment) -> Result<user::User, Error> {
+pub fn read_user(userid: Userid, mut rpcenv: &mut dyn RpcEnvironment) -> Result<user::User, Error> {
     let (config, digest) = user::config()?;
-    let user = config.lookup("user", &userid)?;
+    let user = config.lookup("user", userid.as_str())?;
     rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
     Ok(user)
 }
@@ -140,7 +140,7 @@ pub fn read_user(userid: String, mut rpcenv: &mut dyn RpcEnvironment) -> Result<
     input: {
         properties: {
             userid: {
-                schema: PROXMOX_USER_ID_SCHEMA,
+                type: Userid,
             },
             comment: {
                 optional: true,
@@ -182,7 +182,7 @@ pub fn read_user(userid: String, mut rpcenv: &mut dyn RpcEnvironment) -> Result<
 )]
 /// Update user configuration.
 pub fn update_user(
-    userid: String,
+    userid: Userid,
     comment: Option<String>,
     enable: Option<bool>,
     expire: Option<i64>,
@@ -193,7 +193,7 @@ pub fn update_user(
     digest: Option<String>,
 ) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let (mut config, expected_digest) = user::config()?;
 
@@ -202,7 +202,7 @@ pub fn update_user(
         crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
     }
 
-    let mut data: user::User = config.lookup("user", &userid)?;
+    let mut data: user::User = config.lookup("user", userid.as_str())?;
 
     if let Some(comment) = comment {
         let comment = comment.trim().to_string();
@@ -222,9 +222,8 @@ pub fn update_user(
     }
 
     if let Some(password) = password {
-        let (username, realm) = crate::auth::parse_userid(&userid)?;
-        let authenticator = crate::auth::lookup_authenticator(&realm)?;
-        authenticator.store_password(&username, &password)?;
+        let authenticator = crate::auth::lookup_authenticator(userid.realm())?;
+        authenticator.store_password(userid.name(), &password)?;
     }
 
     if let Some(firstname) = firstname {
@@ -238,7 +237,7 @@ pub fn update_user(
         data.email = if email.is_empty() { None } else { Some(email) };
     }
 
-    config.set_data(&userid, "user", &data)?;
+    config.set_data(userid.as_str(), "user", &data)?;
 
     user::save_config(&config)?;
 
@@ -250,7 +249,7 @@ pub fn update_user(
     input: {
         properties: {
             userid: {
-                schema: PROXMOX_USER_ID_SCHEMA,
+                type: Userid,
             },
             digest: {
                 optional: true,
@@ -263,9 +262,9 @@ pub fn update_user(
     },
 )]
 /// Remove a user from the configuration file.
-pub fn delete_user(userid: String, digest: Option<String>) -> Result<(), Error> {
+pub fn delete_user(userid: Userid, digest: Option<String>) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(user::USER_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let (mut config, expected_digest) = user::config()?;
 
@@ -274,8 +273,8 @@ pub fn delete_user(userid: String, digest: Option<String>) -> Result<(), Error>
         crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
     }
 
-    match config.sections.get(&userid) {
-        Some(_) => { config.sections.remove(&userid); },
+    match config.sections.get(userid.as_str()) {
+        Some(_) => { config.sections.remove(userid.as_str()); },
         None => bail!("user '{}' does not exist.", userid),
     }
 

src/api2/admin.rs

@@ -2,9 +2,11 @@ use proxmox::api::router::{Router, SubdirMap};
 use proxmox::list_subdirs_api_method;
 
 pub mod datastore;
+pub mod sync;
 
 const SUBDIRS: SubdirMap = &[
-    ("datastore", &datastore::ROUTER)
+    ("datastore", &datastore::ROUTER),
+    ("sync", &sync::ROUTER)
 ];
 
 pub const ROUTER: Router = Router::new()

src/api2/admin/sync.rs

@@ -0,0 +1,113 @@
+use anyhow::{format_err, Error};
+use serde_json::Value;
+
+use proxmox::api::{api, ApiMethod, Router, RpcEnvironment};
+use proxmox::api::router::SubdirMap;
+use proxmox::{list_subdirs_api_method, sortable};
+
+use crate::api2::types::*;
+use crate::api2::pull::do_sync_job;
+use crate::config::sync::{self, SyncJobStatus, SyncJobConfig};
+use crate::server::UPID;
+use crate::config::jobstate::{Job, JobState};
+use crate::tools::systemd::time::{
+    parse_calendar_event, compute_next_event};
+
+#[api(
+    input: {
+        properties: {},
+    },
+    returns: {
+        description: "List configured jobs and their status.",
+        type: Array,
+        items: { type: sync::SyncJobStatus },
+    },
+)]
+/// List all sync jobs
+pub fn list_sync_jobs(
+    _param: Value,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<SyncJobStatus>, Error> {
+
+    let (config, digest) = sync::config()?;
+
+    let mut list: Vec<SyncJobStatus> = config.convert_to_typed_array("sync")?;
+
+    for job in &mut list {
+        let last_state = JobState::load("syncjob", &job.id)
+            .map_err(|err| format_err!("could not open statefile for {}: {}", &job.id, err))?;
+        let (upid, endtime, state, starttime) = match last_state {
+            JobState::Created { time } => (None, None, None, time),
+            JobState::Started { upid } => {
+                let parsed_upid: UPID = upid.parse()?;
+                (Some(upid), None, None, parsed_upid.starttime)
+            },
+            JobState::Finished { upid, state } => {
+                let parsed_upid: UPID = upid.parse()?;
+                (Some(upid), Some(state.endtime()), Some(state.to_string()), parsed_upid.starttime)
+            },
+        };
+
+        job.last_run_upid = upid;
+        job.last_run_state = state;
+        job.last_run_endtime = endtime;
+
+        let last = job.last_run_endtime.unwrap_or_else(|| starttime);
+
+        job.next_run = (|| -> Option<i64> {
+            let schedule = job.schedule.as_ref()?;
+            let event = parse_calendar_event(&schedule).ok()?;
+            compute_next_event(&event, last, false).ok()
+        })();
+    }
+
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+    Ok(list)
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            }
+        }
+    }
+)]
+/// Runs the sync jobs manually.
+fn run_sync_job(
+    id: String,
+    _info: &ApiMethod,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+
+    let (config, _digest) = sync::config()?;
+    let sync_job: SyncJobConfig = config.lookup("sync", &id)?;
+
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+
+    let job = Job::new("syncjob", &id)?;
+
+    let upid_str = do_sync_job(job, sync_job, &userid, None)?;
+
+    Ok(upid_str)
+}
+
+#[sortable]
+const SYNC_INFO_SUBDIRS: SubdirMap = &[
+    (
+        "run",
+        &Router::new()
+            .post(&API_METHOD_RUN_SYNC_JOB)
+    ),
+];
+
+const SYNC_INFO_ROUTER: Router = Router::new()
+    .get(&list_subdirs_api_method!(SYNC_INFO_SUBDIRS))
+    .subdirs(SYNC_INFO_SUBDIRS);
+
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_SYNC_JOBS)
+    .match_all("id", &SYNC_INFO_ROUTER);

src/api2/backup.rs

@@ -10,12 +10,13 @@ use proxmox::api::{ApiResponseFuture, ApiHandler, ApiMethod, Router, RpcEnvironm
 use proxmox::api::router::SubdirMap;
 use proxmox::api::schema::*;
 
-use crate::tools::{self, WrappedReaderStream};
+use crate::tools;
 use crate::server::{WorkerTask, H2Service};
 use crate::backup::*;
 use crate::api2::types::*;
 use crate::config::acl::PRIV_DATASTORE_BACKUP;
 use crate::config::cached_user_info::CachedUserInfo;
+use crate::tools::fs::lock_dir_noblock;
 
 mod environment;
 use environment::*;
@@ -56,12 +57,12 @@ fn upgrade_to_backup_protocol(
 async move {
     let debug = param["debug"].as_bool().unwrap_or(false);
 
-    let username = rpcenv.get_user().unwrap();
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
 
     let store = tools::required_string_param(&param, "store")?.to_owned();
 
     let user_info = CachedUserInfo::new()?;
-    user_info.check_privs(&username, &["datastore", &store], PRIV_DATASTORE_BACKUP, false)?;
+    user_info.check_privs(&userid, &["datastore", &store], PRIV_DATASTORE_BACKUP, false)?;
 
     let datastore = DataStore::lookup_datastore(&store)?;
 
@@ -88,30 +89,36 @@ async move {
     let env_type = rpcenv.env_type();
 
     let backup_group = BackupGroup::new(backup_type, backup_id);
-    let owner = datastore.create_backup_group(&backup_group, &username)?;
+
+    // lock backup group to only allow one backup per group at a time
+    let (owner, _group_guard) = datastore.create_locked_backup_group(&backup_group, &userid)?;
+
     // permission check
-    if owner != username { // only the owner is allowed to create additional snapshots
-        bail!("backup owner check failed ({} != {})", username, owner);
+    if owner != userid { // only the owner is allowed to create additional snapshots
+        bail!("backup owner check failed ({} != {})", userid, owner);
     }
 
-    let last_backup = BackupInfo::last_backup(&datastore.base_path(), &backup_group).unwrap_or(None);
-    let backup_dir = BackupDir::new_with_group(backup_group, backup_time);
+    let last_backup = BackupInfo::last_backup(&datastore.base_path(), &backup_group, true).unwrap_or(None);
+    let backup_dir = BackupDir::new_with_group(backup_group.clone(), backup_time);
 
-    if let Some(last) = &last_backup {
+    let _last_guard = if let Some(last) = &last_backup {
         if backup_dir.backup_time() <= last.backup_dir.backup_time() {
             bail!("backup timestamp is older than last backup.");
         }
-        // fixme: abort if last backup is still running - howto test?
-        // Idea: write upid into a file inside snapshot dir. then test if
-        // it is still running here.
-    }
 
-    let (path, is_new) = datastore.create_backup_dir(&backup_dir)?;
-    if !is_new { bail!("backup directorty already exists."); }
+        // lock last snapshot to prevent forgetting/pruning it during backup
+        let full_path = datastore.snapshot_path(&last.backup_dir);
+        Some(lock_dir_noblock(&full_path, "snapshot", "base snapshot is already locked by another operation")?)
+    } else {
+        None
+    };
 
-    WorkerTask::spawn("backup", Some(worker_id), &username.clone(), true, move |worker| {
+    let (path, is_new, _snap_guard) = datastore.create_locked_backup_dir(&backup_dir)?;
+    if !is_new { bail!("backup directory already exists."); }
+
+    WorkerTask::spawn("backup", Some(worker_id), userid.clone(), true, move |worker| {
         let mut env = BackupEnvironment::new(
-            env_type, username.clone(), worker.clone(), datastore, backup_dir);
+            env_type, userid, worker.clone(), datastore, backup_dir);
 
         env.debug = debug;
         env.last_backup = last_backup;
@@ -144,6 +151,11 @@ async move {
             .map(|_| Err(format_err!("task aborted")));
 
         async move {
+            // keep flock until task ends
+            let _group_guard = _group_guard;
+            let _snap_guard = _snap_guard;
+            let _last_guard = _last_guard;
+
             let res = select!{
                 req = req_fut => req,
                 abrt = abort_future => abrt,
@@ -151,7 +163,7 @@ async move {
 
             match (res, env.ensure_finished()) {
                 (Ok(_), Ok(())) => {
-                    env.log("backup finished sucessfully");
+                    env.log("backup finished successfully");
                     Ok(())
                 },
                 (Err(err), Ok(())) => {
@@ -199,7 +211,6 @@ pub const BACKUP_API_SUBDIRS: SubdirMap = &[
     ),
     (
         "dynamic_index", &Router::new()
-            .download(&API_METHOD_DYNAMIC_CHUNK_INDEX)
             .post(&API_METHOD_CREATE_DYNAMIC_INDEX)
             .put(&API_METHOD_DYNAMIC_APPEND)
     ),
@@ -222,10 +233,13 @@ pub const BACKUP_API_SUBDIRS: SubdirMap = &[
     ),
     (
         "fixed_index", &Router::new()
-            .download(&API_METHOD_FIXED_CHUNK_INDEX)
             .post(&API_METHOD_CREATE_FIXED_INDEX)
             .put(&API_METHOD_FIXED_APPEND)
     ),
+    (
+        "previous", &Router::new()
+            .download(&API_METHOD_DOWNLOAD_PREVIOUS)
+    ),
     (
         "speedtest", &Router::new()
             .upload(&API_METHOD_UPLOAD_SPEEDTEST)
@@ -284,6 +298,8 @@ pub const API_METHOD_CREATE_FIXED_INDEX: ApiMethod = ApiMethod::new(
              .minimum(1)
              .schema()
             ),
+            ("reuse-csum", true, &StringSchema::new("If set, compare last backup's \
+                csum and reuse index for incremental backup if it matches.").schema()),
         ]),
     )
 );
@@ -296,10 +312,9 @@ fn create_fixed_index(
 
     let env: &BackupEnvironment = rpcenv.as_ref();
 
-    println!("PARAM: {:?}", param);
-
     let name = tools::required_string_param(&param, "archive-name")?.to_owned();
     let size = tools::required_integer_param(&param, "size")? as usize;
+    let reuse_csum = param["reuse-csum"].as_str();
 
     let archive_name = name.clone();
     if !archive_name.ends_with(".fidx") {
@@ -307,12 +322,49 @@ fn create_fixed_index(
     }
 
     let mut path = env.backup_dir.relative_path();
-    path.push(archive_name);
+    path.push(&archive_name);
 
     let chunk_size = 4096*1024; // todo: ??
 
-    let index = env.datastore.create_fixed_writer(&path, size, chunk_size)?;
-    let wid = env.register_fixed_writer(index, name, size, chunk_size as u32)?;
+    // do incremental backup if csum is set
+    let mut reader = None;
+    let mut incremental = false;
+    if let Some(csum) = reuse_csum {
+        incremental = true;
+        let last_backup = match &env.last_backup {
+            Some(info) => info,
+            None => {
+                bail!("cannot reuse index - no previous backup exists");
+            }
+        };
+
+        let mut last_path = last_backup.backup_dir.relative_path();
+        last_path.push(&archive_name);
+
+        let index = match env.datastore.open_fixed_reader(last_path) {
+            Ok(index) => index,
+            Err(_) => {
+                bail!("cannot reuse index - no previous backup exists for archive");
+            }
+        };
+
+        let (old_csum, _) = index.compute_csum();
+        let old_csum = proxmox::tools::digest_to_hex(&old_csum);
+        if old_csum != csum {
+            bail!("expected csum ({}) doesn't match last backup's ({}), cannot do incremental backup",
+                csum, old_csum);
+        }
+
+        reader = Some(index);
+    }
+
+    let mut writer = env.datastore.create_fixed_writer(&path, size, chunk_size)?;
+
+    if let Some(reader) = reader {
+        writer.clone_data_from(&reader)?;
+    }
+
+    let wid = env.register_fixed_writer(writer, name, size, chunk_size as u32, incremental)?;
 
     env.log(format!("created new fixed index {} ({:?})", wid, path));
 
@@ -378,7 +430,7 @@ fn dynamic_append (
 
         env.dynamic_writer_append_chunk(wid, offset, size, &digest)?;
 
-        env.debug(format!("sucessfully added chunk {} to dynamic index {} (offset {}, size {})", digest_str, wid, offset, size));
+        env.debug(format!("successfully added chunk {} to dynamic index {} (offset {}, size {})", digest_str, wid, offset, size));
     }
 
     Ok(Value::Null)
@@ -443,7 +495,7 @@ fn fixed_append (
 
         env.fixed_writer_append_chunk(wid, offset, size, &digest)?;
 
-        env.debug(format!("sucessfully added chunk {} to fixed index {} (offset {}, size {})", digest_str, wid, offset, size));
+        env.debug(format!("successfully added chunk {} to fixed index {} (offset {}, size {})", digest_str, wid, offset, size));
     }
 
     Ok(Value::Null)
@@ -498,7 +550,7 @@ fn close_dynamic_index (
 
     env.dynamic_writer_close(wid, chunk_count, size, csum)?;
 
-    env.log(format!("sucessfully closed dynamic index {}", wid));
+    env.log(format!("successfully closed dynamic index {}", wid));
 
     Ok(Value::Null)
 }
@@ -520,15 +572,15 @@ pub const API_METHOD_CLOSE_FIXED_INDEX: ApiMethod = ApiMethod::new(
             (
                 "chunk-count",
                 false,
-                &IntegerSchema::new("Chunk count. This is used to verify that the server got all chunks.")
-                    .minimum(1)
+                &IntegerSchema::new("Chunk count. This is used to verify that the server got all chunks. Ignored for incremental backups.")
+                    .minimum(0)
                     .schema()
             ),
             (
                 "size",
                 false,
-                &IntegerSchema::new("File size. This is used to verify that the server got all data.")
-                    .minimum(1)
+                &IntegerSchema::new("File size. This is used to verify that the server got all data. Ignored for incremental backups.")
+                    .minimum(0)
                     .schema()
             ),
             ("csum", false, &StringSchema::new("Digest list checksum.").schema()),
@@ -552,7 +604,7 @@ fn close_fixed_index (
 
     env.fixed_writer_close(wid, chunk_count, size, csum)?;
 
-    env.log(format!("sucessfully closed fixed index {}", wid));
+    env.log(format!("successfully closed fixed index {}", wid));
 
     Ok(Value::Null)
 }
@@ -566,26 +618,23 @@ fn finish_backup (
     let env: &BackupEnvironment = rpcenv.as_ref();
 
     env.finish_backup()?;
-    env.log("sucessfully finished backup");
+    env.log("successfully finished backup");
 
     Ok(Value::Null)
 }
 
 #[sortable]
-pub const API_METHOD_DYNAMIC_CHUNK_INDEX: ApiMethod = ApiMethod::new(
-    &ApiHandler::AsyncHttp(&dynamic_chunk_index),
+pub const API_METHOD_DOWNLOAD_PREVIOUS: ApiMethod = ApiMethod::new(
+    &ApiHandler::AsyncHttp(&download_previous),
     &ObjectSchema::new(
-        r###"
-Download the dynamic chunk index from the previous backup.
-Simply returns an empty list if this is the first backup.
-"### ,
+        "Download archive from previous backup.",
         &sorted!([
             ("archive-name", false, &crate::api2::types::BACKUP_ARCHIVE_NAME_SCHEMA)
         ]),
     )
 );
 
-fn dynamic_chunk_index(
+fn download_previous(
     _parts: Parts,
     _req_body: Body,
     param: Value,
@@ -598,130 +647,38 @@ fn dynamic_chunk_index(
 
         let archive_name = tools::required_string_param(&param, "archive-name")?.to_owned();
 
-        if !archive_name.ends_with(".didx") {
-            bail!("wrong archive extension: '{}'", archive_name);
-        }
-
-        let empty_response = {
-            Response::builder()
-                .status(StatusCode::OK)
-                .body(Body::empty())?
-        };
-
         let last_backup = match &env.last_backup {
             Some(info) => info,
-            None => return Ok(empty_response),
+            None => bail!("no previous backup"),
         };
 
-        let mut path = last_backup.backup_dir.relative_path();
+        let mut path = env.datastore.snapshot_path(&last_backup.backup_dir);
         path.push(&archive_name);
 
-        let index = match env.datastore.open_dynamic_reader(path) {
-            Ok(index) => index,
-            Err(_) => {
-                env.log(format!("there is no last backup for archive '{}'", archive_name));
-                return Ok(empty_response);
+        {
+            let index: Option<Box<dyn IndexFile>> = match archive_type(&archive_name)? {
+                ArchiveType::FixedIndex => {
+                    let index = env.datastore.open_fixed_reader(&path)?;
+                    Some(Box::new(index))
+                }
+                ArchiveType::DynamicIndex => {
+                    let index = env.datastore.open_dynamic_reader(&path)?;
+                    Some(Box::new(index))
+                }
+                _ => { None }
+            };
+            if let Some(index) = index {
+                env.log(format!("register chunks in '{}' from previous backup.", archive_name));
+
+                for pos in 0..index.index_count() {
+                    let info = index.chunk_info(pos).unwrap();
+                    let size = info.range.end - info.range.start;
+                    env.register_chunk(info.digest, size as u32)?;
+                }
             }
-        };
-
-        env.log(format!("download last backup index for archive '{}'", archive_name));
-
-        let count = index.index_count();
-        for pos in 0..count {
-            let (start, end, digest) = index.chunk_info(pos)?;
-            let size = (end - start) as u32;
-            env.register_chunk(digest, size)?;
         }
 
-        let reader = DigestListEncoder::new(Box::new(index));
-
-        let stream = WrappedReaderStream::new(reader);
-
-        // fixme: set size, content type?
-        let response = http::Response::builder()
-            .status(200)
-            .body(Body::wrap_stream(stream))?;
-
-        Ok(response)
-    }.boxed()
-}
-
-#[sortable]
-pub const API_METHOD_FIXED_CHUNK_INDEX: ApiMethod = ApiMethod::new(
-    &ApiHandler::AsyncHttp(&fixed_chunk_index),
-    &ObjectSchema::new(
-        r###"
-Download the fixed chunk index from the previous backup.
-Simply returns an empty list if this is the first backup.
-"### ,
-        &sorted!([
-            ("archive-name", false, &crate::api2::types::BACKUP_ARCHIVE_NAME_SCHEMA)
-        ]),
-    )
-);
-
-fn fixed_chunk_index(
-    _parts: Parts,
-    _req_body: Body,
-    param: Value,
-    _info: &ApiMethod,
-    rpcenv: Box<dyn RpcEnvironment>,
-) -> ApiResponseFuture {
-
-    async move {
-        let env: &BackupEnvironment = rpcenv.as_ref();
-
-        let archive_name = tools::required_string_param(&param, "archive-name")?.to_owned();
-
-        if !archive_name.ends_with(".fidx") {
-            bail!("wrong archive extension: '{}'", archive_name);
-        }
-
-        let empty_response = {
-            Response::builder()
-                .status(StatusCode::OK)
-                .body(Body::empty())?
-        };
-
-        let last_backup = match &env.last_backup {
-            Some(info) => info,
-            None => return Ok(empty_response),
-        };
-
-        let mut path = last_backup.backup_dir.relative_path();
-        path.push(&archive_name);
-
-        let index = match env.datastore.open_fixed_reader(path) {
-            Ok(index) => index,
-            Err(_) => {
-                env.log(format!("there is no last backup for archive '{}'", archive_name));
-                return Ok(empty_response);
-            }
-        };
-
-        env.log(format!("download last backup index for archive '{}'", archive_name));
-
-        let count = index.index_count();
-        let image_size = index.index_bytes();
-        for pos in 0..count {
-            let digest = index.index_digest(pos).unwrap();
-            // Note: last chunk can be smaller
-            let start = (pos*index.chunk_size) as u64;
-            let mut end = start + index.chunk_size as u64;
-            if end > image_size { end = image_size; }
-            let size = (end - start) as u32;
-            env.register_chunk(*digest, size)?;
-        }
-
-        let reader = DigestListEncoder::new(Box::new(index));
-
-        let stream = WrappedReaderStream::new(reader);
-
-        // fixme: set size, content type?
-        let response = http::Response::builder()
-            .status(200)
-            .body(Body::wrap_stream(stream))?;
-
-        Ok(response)
+        env.log(format!("download '{}' from previous backup.", archive_name));
+        crate::api2::helpers::create_download_response(path).await
     }.boxed()
 }

src/api2/backup/environment.rs

@@ -1,18 +1,21 @@
-use anyhow::{bail, Error};
+use anyhow::{bail, format_err, Error};
 use std::sync::{Arc, Mutex};
 use std::collections::HashMap;
 
+use ::serde::{Serialize};
 use serde_json::{json, Value};
 
 use proxmox::tools::digest_to_hex;
 use proxmox::tools::fs::{replace_file, CreateOptions};
 use proxmox::api::{RpcEnvironment, RpcEnvironmentType};
 
-use crate::server::WorkerTask;
+use crate::api2::types::Userid;
 use crate::backup::*;
+use crate::server::WorkerTask;
 use crate::server::formatter::*;
 use hyper::{Body, Response};
 
+#[derive(Copy, Clone, Serialize)]
 struct UploadStatistic {
     count: u64,
     size: u64,
@@ -31,6 +34,19 @@ impl UploadStatistic {
     }
 }
 
+impl std::ops::Add for UploadStatistic {
+    type Output = Self;
+
+    fn add(self, other: Self) -> Self {
+        Self {
+            count: self.count + other.count,
+            size: self.size + other.size,
+            compressed_size: self.compressed_size + other.compressed_size,
+            duplicates: self.duplicates + other.duplicates,
+        }
+    }
+}
+
 struct DynamicWriterState {
     name: String,
     index: DynamicIndexWriter,
@@ -47,15 +63,18 @@ struct FixedWriterState {
     chunk_count: u64,
     small_chunk_count: usize, // allow 0..1 small chunks (last chunk may be smaller)
     upload_stat: UploadStatistic,
+    incremental: bool,
 }
 
 struct SharedBackupState {
     finished: bool,
     uid_counter: usize,
-    file_counter: usize, // sucessfully uploaded files
+    file_counter: usize, // successfully uploaded files
     dynamic_writers: HashMap<usize, DynamicWriterState>,
     fixed_writers: HashMap<usize, FixedWriterState>,
     known_chunks: HashMap<[u8;32], u32>,
+    backup_size: u64, // sums up size of all files
+    backup_stat: UploadStatistic,
 }
 
 impl SharedBackupState {
@@ -81,7 +100,7 @@ impl SharedBackupState {
 pub struct BackupEnvironment {
     env_type: RpcEnvironmentType,
     result_attributes: Value,
-    user: String,
+    user: Userid,
     pub debug: bool,
     pub formatter: &'static OutputFormatter,
     pub worker: Arc<WorkerTask>,
@@ -94,7 +113,7 @@ pub struct BackupEnvironment {
 impl BackupEnvironment {
     pub fn new(
         env_type: RpcEnvironmentType,
-        user: String,
+        user: Userid,
         worker: Arc<WorkerTask>,
         datastore: Arc<DataStore>,
         backup_dir: BackupDir,
@@ -107,6 +126,8 @@ impl BackupEnvironment {
             dynamic_writers: HashMap::new(),
             fixed_writers: HashMap::new(),
             known_chunks: HashMap::new(),
+            backup_size: 0,
+            backup_stat: UploadStatistic::new(),
         };
 
         Self {
@@ -237,7 +258,7 @@ impl BackupEnvironment {
     }
 
     /// Store the writer with an unique ID
-    pub fn register_fixed_writer(&self, index: FixedIndexWriter, name: String, size: usize, chunk_size: u32) -> Result<usize, Error> {
+    pub fn register_fixed_writer(&self, index: FixedIndexWriter, name: String, size: usize, chunk_size: u32, incremental: bool) -> Result<usize, Error> {
         let mut state = self.state.lock().unwrap();
 
         state.ensure_unfinished()?;
@@ -245,7 +266,7 @@ impl BackupEnvironment {
         let uid = state.next_uid();
 
         state.fixed_writers.insert(uid, FixedWriterState {
-            index, name, chunk_count: 0, size, chunk_size, small_chunk_count: 0, upload_stat: UploadStatistic::new(),
+            index, name, chunk_count: 0, size, chunk_size, small_chunk_count: 0, upload_stat: UploadStatistic::new(), incremental,
         });
 
         Ok(uid)
@@ -310,7 +331,13 @@ impl BackupEnvironment {
 
         self.log(format!("Upload size: {} ({}%)", upload_stat.size, (upload_stat.size*100)/size));
 
-        let client_side_duplicates = chunk_count - upload_stat.count;
+        // account for zero chunk, which might be uploaded but never used
+        let client_side_duplicates = if chunk_count < upload_stat.count {
+            0
+        } else {
+            chunk_count - upload_stat.count
+        };
+
         let server_side_duplicates = upload_stat.duplicates;
 
         if (client_side_duplicates + server_side_duplicates) > 0 {
@@ -346,7 +373,6 @@ impl BackupEnvironment {
 
         let expected_csum = data.index.close()?;
 
-        println!("server checksum {:?} client: {:?}", expected_csum, csum);
         if csum != expected_csum {
             bail!("dynamic writer '{}' close failed - got unexpected checksum", data.name);
         }
@@ -354,6 +380,8 @@ impl BackupEnvironment {
         self.log_upload_stat(&data.name, &csum, &uuid, size, chunk_count, &data.upload_stat);
 
         state.file_counter += 1;
+        state.backup_size += size;
+        state.backup_stat = state.backup_stat + data.upload_stat;
 
         Ok(())
     }
@@ -373,21 +401,21 @@ impl BackupEnvironment {
             bail!("fixed writer '{}' close failed - received wrong number of chunk ({} != {})", data.name, data.chunk_count, chunk_count);
         }
 
-        let expected_count = data.index.index_length();
+        if !data.incremental {
+            let expected_count = data.index.index_length();
 
-        if chunk_count != (expected_count as u64) {
-            bail!("fixed writer '{}' close failed - unexpected chunk count ({} != {})", data.name, expected_count, chunk_count);
-        }
+            if chunk_count != (expected_count as u64) {
+                bail!("fixed writer '{}' close failed - unexpected chunk count ({} != {})", data.name, expected_count, chunk_count);
+            }
 
-        if size != (data.size as u64) {
-            bail!("fixed writer '{}' close failed - unexpected file size ({} != {})", data.name, data.size, size);
+            if size != (data.size as u64) {
+                bail!("fixed writer '{}' close failed - unexpected file size ({} != {})", data.name, data.size, size);
+            }
         }
 
         let uuid = data.index.uuid;
-
         let expected_csum = data.index.close()?;
 
-        println!("server checksum {:?} client: {:?}", expected_csum, csum);
         if csum != expected_csum {
             bail!("fixed writer '{}' close failed - got unexpected checksum", data.name);
         }
@@ -395,6 +423,8 @@ impl BackupEnvironment {
         self.log_upload_stat(&data.name, &expected_csum, &uuid, size, chunk_count, &data.upload_stat);
 
         state.file_counter += 1;
+        state.backup_size += size;
+        state.backup_stat = state.backup_stat + data.upload_stat;
 
         Ok(())
     }
@@ -408,9 +438,8 @@ impl BackupEnvironment {
         let blob_len = data.len();
         let orig_len = data.len(); // fixme:
 
-        let blob = DataBlob::from_raw(data)?;
-        // always verify CRC at server side
-        blob.verify_crc()?;
+        // always verify blob/CRC at server side
+        let blob = DataBlob::load_from_reader(&mut &data[..])?;
 
         let raw_data = blob.raw_data();
         replace_file(&path, raw_data, CreateOptions::new())?;
@@ -419,6 +448,8 @@ impl BackupEnvironment {
 
         let mut state = self.state.lock().unwrap();
         state.file_counter += 1;
+        state.backup_size += orig_len as u64;
+        state.backup_stat.size += blob_len as u64;
 
         Ok(())
     }
@@ -430,8 +461,6 @@ impl BackupEnvironment {
 
         state.ensure_unfinished()?;
 
-        state.finished = true;
-
         if state.dynamic_writers.len() != 0 {
             bail!("found open index writer - unable to finish backup");
         }
@@ -440,6 +469,30 @@ impl BackupEnvironment {
             bail!("backup does not contain valid files (file count == 0)");
         }
 
+        // check manifest
+        let mut manifest = self.datastore.load_manifest_json(&self.backup_dir)
+            .map_err(|err| format_err!("unable to load manifest blob - {}", err))?;
+
+        let stats = serde_json::to_value(state.backup_stat)?;
+
+        manifest["unprotected"]["chunk_upload_stats"] = stats;
+
+        self.datastore.store_manifest(&self.backup_dir, manifest)
+            .map_err(|err| format_err!("unable to store manifest blob - {}", err))?;
+
+        if let Some(base) = &self.last_backup {
+            let path = self.datastore.snapshot_path(&base.backup_dir);
+            if !path.exists() {
+                bail!(
+                    "base snapshot {} was removed during backup, cannot finish as chunks might be missing",
+                    base.backup_dir
+                );
+            }
+        }
+
+        // marks the backup as successful
+        state.finished = true;
+
         Ok(())
     }
 
@@ -472,7 +525,7 @@ impl BackupEnvironment {
         let mut state = self.state.lock().unwrap();
         state.finished = true;
 
-        self.datastore.remove_backup_dir(&self.backup_dir)?;
+        self.datastore.remove_backup_dir(&self.backup_dir, true)?;
 
         Ok(())
     }
@@ -497,7 +550,7 @@ impl RpcEnvironment for BackupEnvironment {
     }
 
     fn get_user(&self) -> Option<String> {
-        Some(self.user.clone())
+        Some(self.user.to_string())
     }
 }
 

src/api2/backup/upload_chunk.rs

@@ -243,7 +243,7 @@ pub const API_METHOD_UPLOAD_BLOB: ApiMethod = ApiMethod::new(
         &sorted!([
             ("file-name", false, &crate::api2::types::BACKUP_ARCHIVE_NAME_SCHEMA),
             ("encoded-size", false, &IntegerSchema::new("Encoded blob size.")
-             .minimum((std::mem::size_of::<DataBlobHeader>() as isize) +1)
+             .minimum(std::mem::size_of::<DataBlobHeader>() as isize)
              .maximum(1024*1024*16+(std::mem::size_of::<EncryptedDataBlobHeader>() as isize))
              .schema()
             )

src/api2/config/datastore.rs

@@ -5,6 +5,7 @@ use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, Router, RpcEnvironment, Permission};
+use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
 use crate::backup::*;
@@ -99,7 +100,7 @@ pub fn list_datastores(
 /// Create new datastore config.
 pub fn create_datastore(param: Value) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let datastore: datastore::DataStoreConfig = serde_json::from_value(param.clone())?;
 
@@ -253,7 +254,7 @@ pub fn update_datastore(
     digest: Option<String>,
 ) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     // pass/compare digest
     let (mut config, expected_digest) = datastore::config()?;
@@ -327,7 +328,7 @@ pub fn update_datastore(
 /// Remove a datastore configuration.
 pub fn delete_datastore(name: String, digest: Option<String>) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let (mut config, expected_digest) = datastore::config()?;
 

src/api2/config/remote.rs

@@ -4,6 +4,7 @@ use ::serde::{Deserialize, Serialize};
 use base64;
 
 use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
+use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
 use crate::config::remote;
@@ -60,7 +61,7 @@ pub fn list_remotes(
                 schema: DNS_NAME_OR_IP_SCHEMA,
             },
             userid: {
-                schema: PROXMOX_USER_ID_SCHEMA,
+                type: Userid,
             },
             password: {
                 schema: remote::REMOTE_PASSWORD_SCHEMA,
@@ -78,7 +79,7 @@ pub fn list_remotes(
 /// Create new remote.
 pub fn create_remote(password: String, param: Value) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let mut data = param.clone();
     data["password"] = Value::from(base64::encode(password.as_bytes()));
@@ -154,7 +155,7 @@ pub enum DeletableProperty {
             },
             userid: {
                 optional: true,
-               schema: PROXMOX_USER_ID_SCHEMA,
+                type: Userid,
             },
             password: {
                 optional: true,
@@ -187,14 +188,14 @@ pub fn update_remote(
     name: String,
     comment: Option<String>,
     host: Option<String>,
-    userid: Option<String>,
+    userid: Option<Userid>,
     password: Option<String>,
     fingerprint: Option<String>,
     delete: Option<Vec<DeletableProperty>>,
     digest: Option<String>,
 ) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let (mut config, expected_digest) = remote::config()?;
 
@@ -255,7 +256,7 @@ pub fn update_remote(
 /// Remove a remote from the configuration file.
 pub fn delete_remote(name: String, digest: Option<String>) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(remote::REMOTE_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let (mut config, expected_digest) = remote::config()?;
 
@@ -269,6 +270,8 @@ pub fn delete_remote(name: String, digest: Option<String>) -> Result<(), Error>
         None => bail!("remote '{}' does not exist.", name),
     }
 
+    remote::save_config(&config)?;
+
     Ok(())
 }
 

src/api2/config/sync.rs

@@ -3,6 +3,7 @@ use serde_json::Value;
 use ::serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, Router, RpcEnvironment};
+use proxmox::tools::fs::open_file_locked;
 
 use crate::api2::types::*;
 use crate::config::sync::{self, SyncJobConfig};
@@ -60,7 +61,7 @@ pub fn list_sync_jobs(
             },
             schedule: {
                 optional: true,
-                schema: GC_SCHEDULE_SCHEMA,
+                schema: SYNC_SCHEDULE_SCHEMA,
             },
         },
     },
@@ -68,7 +69,7 @@ pub fn list_sync_jobs(
 /// Create a new sync job.
 pub fn create_sync_job(param: Value) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let sync_job: sync::SyncJobConfig = serde_json::from_value(param.clone())?;
 
@@ -82,6 +83,8 @@ pub fn create_sync_job(param: Value) -> Result<(), Error> {
 
     sync::save_config(&config)?;
 
+    crate::config::jobstate::create_state_file("syncjob", &sync_job.id)?;
+
     Ok(())
 }
 
@@ -154,7 +157,7 @@ pub enum DeletableProperty {
             },
             schedule: {
                 optional: true,
-                schema: GC_SCHEDULE_SCHEMA,
+                schema: SYNC_SCHEDULE_SCHEMA,
             },
             delete: {
                 description: "List of properties to delete.",
@@ -184,7 +187,7 @@ pub fn update_sync_job(
     digest: Option<String>,
 ) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     // pass/compare digest
     let (mut config, expected_digest) = sync::config()?;
@@ -247,7 +250,7 @@ pub fn update_sync_job(
 /// Remove a sync job configuration
 pub fn delete_sync_job(id: String, digest: Option<String>) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(sync::SYNC_CFG_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let (mut config, expected_digest) = sync::config()?;
 
@@ -263,6 +266,8 @@ pub fn delete_sync_job(id: String, digest: Option<String>) -> Result<(), Error>
 
     sync::save_config(&config)?;
 
+    crate::config::jobstate::remove_state_file("syncjob", &id)?;
+
     Ok(())
 }
 
@@ -274,4 +279,4 @@ const ITEM_ROUTER: Router = Router::new()
 pub const ROUTER: Router = Router::new()
     .get(&API_METHOD_LIST_SYNC_JOBS)
     .post(&API_METHOD_CREATE_SYNC_JOB)
-    .match_all("name", &ITEM_ROUTER);
+    .match_all("id", &ITEM_ROUTER);

src/api2/helpers.rs

@@ -0,0 +1,29 @@
+use std::path::PathBuf;
+
+use anyhow::Error;
+use futures::stream::TryStreamExt;
+use hyper::{Body, Response, StatusCode, header};
+
+use proxmox::http_bail;
+
+pub async fn create_download_response(path: PathBuf) -> Result<Response<Body>, Error> {
+    let file = match tokio::fs::File::open(path.clone()).await {
+        Ok(file) => file,
+        Err(ref err) if err.kind() == std::io::ErrorKind::NotFound => {
+            http_bail!(NOT_FOUND, "open file {:?} failed - not found", path);
+        }
+        Err(err) => http_bail!(BAD_REQUEST, "open file {:?} failed: {}", path, err),
+    };
+
+    let payload = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
+        .map_ok(|bytes| hyper::body::Bytes::from(bytes.freeze()));
+
+    let body = Body::wrap_stream(payload);
+
+    // fixme: set other headers ?
+    Ok(Response::builder()
+        .status(StatusCode::OK)
+        .header(header::CONTENT_TYPE, "application/octet-stream")
+        .body(body)
+        .unwrap())
+}

src/api2/node.rs

@@ -1,26 +1,324 @@
-use proxmox::api::router::{Router, SubdirMap};
-use proxmox::list_subdirs_api_method;
+use std::net::TcpListener;
+use std::os::unix::io::AsRawFd;
 
-pub mod tasks;
-mod time;
-pub mod network;
+use anyhow::{bail, format_err, Error};
+use futures::future::{FutureExt, TryFutureExt};
+use hyper::body::Body;
+use hyper::http::request::Parts;
+use hyper::upgrade::Upgraded;
+use nix::fcntl::{fcntl, FcntlArg, FdFlag};
+use serde_json::{json, Value};
+use tokio::io::{AsyncBufReadExt, BufReader};
+
+use proxmox::api::router::{Router, SubdirMap};
+use proxmox::api::{
+    api, schema::*, ApiHandler, ApiMethod, ApiResponseFuture, Permission, RpcEnvironment,
+};
+use proxmox::list_subdirs_api_method;
+use proxmox::tools::websocket::WebSocket;
+use proxmox::{identity, sortable};
+
+use crate::api2::types::*;
+use crate::config::acl::PRIV_SYS_CONSOLE;
+use crate::server::WorkerTask;
+use crate::tools;
+use crate::tools::ticket::{self, Empty, Ticket};
+
+pub mod disks;
 pub mod dns;
-mod syslog;
+pub mod network;
+pub mod tasks;
+
+pub(crate) mod rrd;
+
+mod apt;
 mod journal;
 mod services;
 mod status;
-mod rrd;
+mod subscription;
+mod syslog;
+mod time;
+
+pub const SHELL_CMD_SCHEMA: Schema = StringSchema::new("The command to run.")
+    .format(&ApiStringFormat::Enum(&[
+        EnumEntry::new("login", "Login"),
+        EnumEntry::new("upgrade", "Upgrade"),
+    ]))
+    .schema();
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            cmd: {
+                schema: SHELL_CMD_SCHEMA,
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        type: Object,
+        description: "Object with the user, ticket, port and upid",
+        properties: {
+            user: {
+                description: "",
+                type: String,
+            },
+            ticket: {
+                description: "",
+                type: String,
+            },
+            port: {
+                description: "",
+                type: String,
+            },
+            upid: {
+                description: "",
+                type: String,
+            },
+        }
+    },
+    access: {
+        description: "Restricted to users on realm 'pam'",
+        permission: &Permission::Privilege(&["system"], PRIV_SYS_CONSOLE, false),
+    }
+)]
+/// Call termproxy and return shell ticket
+async fn termproxy(
+    cmd: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Value, Error> {
+    let userid: Userid = rpcenv
+        .get_user()
+        .ok_or_else(|| format_err!("unknown user"))?
+        .parse()?;
+
+    if userid.realm() != "pam" {
+        bail!("only pam users can use the console");
+    }
+
+    let path = "/system";
+
+    // use port 0 and let the kernel decide which port is free
+    let listener = TcpListener::bind("localhost:0")?;
+    let port = listener.local_addr()?.port();
+
+    let ticket = Ticket::new(ticket::TERM_PREFIX, &Empty)?
+        .sign(
+            crate::auth_helpers::private_auth_key(),
+            Some(&ticket::term_aad(&userid, &path, port)),
+        )?;
+
+    let mut command = Vec::new();
+    match cmd.as_ref().map(|x| x.as_str()) {
+        Some("login") | None => {
+            command.push("login");
+            if userid == "root@pam" {
+                command.push("-f");
+                command.push("root");
+            }
+        }
+        Some("upgrade") => {
+            if userid != "root@pam" {
+                bail!("only root@pam can upgrade");
+            }
+            // TODO: add nicer/safer wrapper like in PVE instead
+            command.push("sh");
+            command.push("-c");
+            command.push("apt full-upgrade; bash -l");
+        }
+        _ => bail!("invalid command"),
+    };
+
+    let username = userid.name().to_owned();
+    let upid = WorkerTask::spawn(
+        "termproxy",
+        None,
+        userid,
+        false,
+        move |worker| async move {
+            // move inside the worker so that it survives and does not close the port
+            // remove CLOEXEC from listenere so that we can reuse it in termproxy
+            let fd = listener.as_raw_fd();
+            let mut flags = match fcntl(fd, FcntlArg::F_GETFD) {
+                Ok(bits) => FdFlag::from_bits_truncate(bits),
+                Err(err) => bail!("could not get fd: {}", err),
+            };
+            flags.remove(FdFlag::FD_CLOEXEC);
+            if let Err(err) = fcntl(fd, FcntlArg::F_SETFD(flags)) {
+                bail!("could not set fd: {}", err);
+            }
+
+            let mut arguments: Vec<&str> = Vec::new();
+            let fd_string = fd.to_string();
+            arguments.push(&fd_string);
+            arguments.extend_from_slice(&[
+                "--path",
+                &path,
+                "--perm",
+                "Sys.Console",
+                "--authport",
+                "82",
+                "--port-as-fd",
+                "--",
+            ]);
+            arguments.extend_from_slice(&command);
+
+            let mut cmd = tokio::process::Command::new("/usr/bin/termproxy");
+
+            cmd.args(&arguments)
+                .stdout(std::process::Stdio::piped())
+                .stderr(std::process::Stdio::piped());
+
+            let mut child = cmd.spawn().expect("error executing termproxy");
+
+            let stdout = child.stdout.take().expect("no child stdout handle");
+            let stderr = child.stderr.take().expect("no child stderr handle");
+
+            let worker_stdout = worker.clone();
+            let stdout_fut = async move {
+                let mut reader = BufReader::new(stdout).lines();
+                while let Some(line) = reader.next_line().await? {
+                    worker_stdout.log(line);
+                }
+                Ok::<(), Error>(())
+            };
+
+            let worker_stderr = worker.clone();
+            let stderr_fut = async move {
+                let mut reader = BufReader::new(stderr).lines();
+                while let Some(line) = reader.next_line().await? {
+                    worker_stderr.warn(line);
+                }
+                Ok::<(), Error>(())
+            };
+
+            let mut needs_kill = false;
+            let res = tokio::select!{
+                res = &mut child => {
+                    let exit_code = res?;
+                    if !exit_code.success() {
+                        match exit_code.code() {
+                            Some(code) => bail!("termproxy exited with {}", code),
+                            None => bail!("termproxy exited by signal"),
+                        }
+                    }
+                    Ok(())
+                },
+                res = stdout_fut => res,
+                res = stderr_fut => res,
+                res = worker.abort_future() => {
+                    needs_kill = true;
+                    res.map_err(Error::from)
+                }
+            };
+
+            if needs_kill {
+                if res.is_ok() {
+                    child.kill()?;
+                    child.await?;
+                    return Ok(());
+                }
+
+                if let Err(err) = child.kill() {
+                    worker.warn(format!("error killing termproxy: {}", err));
+                } else if let Err(err) = child.await {
+                    worker.warn(format!("error awaiting termproxy: {}", err));
+                }
+            }
+
+            res
+        },
+    )?;
+
+    // FIXME: We're returning the user NAME only?
+    Ok(json!({
+        "user": username,
+        "ticket": ticket,
+        "port": port,
+        "upid": upid,
+    }))
+}
+
+#[sortable]
+pub const API_METHOD_WEBSOCKET: ApiMethod = ApiMethod::new(
+    &ApiHandler::AsyncHttp(&upgrade_to_websocket),
+    &ObjectSchema::new(
+        "Upgraded to websocket",
+        &sorted!([
+            ("node", false, &NODE_SCHEMA),
+            (
+                "vncticket",
+                false,
+                &StringSchema::new("Terminal ticket").schema()
+            ),
+            ("port", false, &IntegerSchema::new("Terminal port").schema()),
+        ]),
+    ),
+)
+.access(
+    Some("The user needs Sys.Console on /system."),
+    &Permission::Privilege(&["system"], PRIV_SYS_CONSOLE, false),
+);
+
+fn upgrade_to_websocket(
+    parts: Parts,
+    req_body: Body,
+    param: Value,
+    _info: &ApiMethod,
+    rpcenv: Box<dyn RpcEnvironment>,
+) -> ApiResponseFuture {
+    async move {
+        let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+        let ticket = tools::required_string_param(&param, "vncticket")?;
+        let port: u16 = tools::required_integer_param(&param, "port")? as u16;
+
+        // will be checked again by termproxy
+        Ticket::<Empty>::parse(ticket)?
+            .verify(
+                crate::auth_helpers::public_auth_key(),
+                ticket::TERM_PREFIX,
+                Some(&ticket::term_aad(&userid, "/system", port)),
+            )?;
+
+        let (ws, response) = WebSocket::new(parts.headers)?;
+
+        crate::server::spawn_internal_task(async move {
+            let conn: Upgraded = match req_body.on_upgrade().map_err(Error::from).await {
+                Ok(upgraded) => upgraded,
+                _ => bail!("error"),
+            };
+
+            let local = tokio::net::TcpStream::connect(format!("localhost:{}", port)).await?;
+            ws.serve_connection(conn, local).await
+        });
+
+        Ok(response)
+    }
+    .boxed()
+}
 
 pub const SUBDIRS: SubdirMap = &[
+    ("apt", &apt::ROUTER),
+    ("disks", &disks::ROUTER),
     ("dns", &dns::ROUTER),
     ("journal", &journal::ROUTER),
     ("network", &network::ROUTER),
     ("rrd", &rrd::ROUTER),
     ("services", &services::ROUTER),
     ("status", &status::ROUTER),
+    ("subscription", &subscription::ROUTER),
     ("syslog", &syslog::ROUTER),
     ("tasks", &tasks::ROUTER),
+    ("termproxy", &Router::new().post(&API_METHOD_TERMPROXY)),
     ("time", &time::ROUTER),
+    (
+        "vncwebsocket",
+        &Router::new().upgrade(&API_METHOD_WEBSOCKET),
+    ),
 ];
 
 pub const ROUTER: Router = Router::new()

src/api2/node/apt.rs

@@ -0,0 +1,268 @@
+use apt_pkg_native::Cache;
+use anyhow::{Error, bail};
+use serde_json::{json, Value};
+
+use proxmox::{list_subdirs_api_method, const_regex};
+use proxmox::api::{api, RpcEnvironment, RpcEnvironmentType, Permission};
+use proxmox::api::router::{Router, SubdirMap};
+
+use crate::server::WorkerTask;
+
+use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
+use crate::api2::types::{APTUpdateInfo, NODE_SCHEMA, Userid, UPID_SCHEMA};
+
+const_regex! {
+    VERSION_EPOCH_REGEX = r"^\d+:";
+    FILENAME_EXTRACT_REGEX = r"^.*/.*?_(.*)_Packages$";
+}
+
+// FIXME: Replace with call to 'apt changelog <pkg> --print-uris'. Currently
+// not possible as our packages do not have a URI set in their Release file
+fn get_changelog_url(
+    package: &str,
+    filename: &str,
+    source_pkg: &str,
+    version: &str,
+    source_version: &str,
+    origin: &str,
+    component: &str,
+) -> Result<String, Error> {
+    if origin == "" {
+        bail!("no origin available for package {}", package);
+    }
+
+    if origin == "Debian" {
+        let source_version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(source_version, "");
+
+        let prefix = if source_pkg.starts_with("lib") {
+            source_pkg.get(0..4)
+        } else {
+            source_pkg.get(0..1)
+        };
+
+        let prefix = match prefix {
+            Some(p) => p,
+            None => bail!("cannot get starting characters of package name '{}'", package)
+        };
+
+        // note: security updates seem to not always upload a changelog for
+        // their package version, so this only works *most* of the time
+        return Ok(format!("https://metadata.ftp-master.debian.org/changelogs/main/{}/{}/{}_{}_changelog",
+                          prefix, source_pkg, source_pkg, source_version));
+
+    } else if origin == "Proxmox" {
+        let version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(version, "");
+
+        let base = match (FILENAME_EXTRACT_REGEX.regex_obj)().captures(filename) {
+            Some(captures) => {
+                let base_capture = captures.get(1);
+                match base_capture {
+                    Some(base_underscore) => base_underscore.as_str().replace("_", "/"),
+                    None => bail!("incompatible filename, cannot find regex group")
+                }
+            },
+            None => bail!("incompatible filename, doesn't match regex")
+        };
+
+        return Ok(format!("http://download.proxmox.com/{}/{}_{}.changelog",
+                          base, package, version));
+    }
+
+    bail!("unknown origin ({}) or component ({})", origin, component)
+}
+
+fn list_installed_apt_packages<F: Fn(&str, &str, &str) -> bool>(filter: F)
+    -> Vec<APTUpdateInfo> {
+
+    let mut ret = Vec::new();
+
+    // note: this is not an 'apt update', it just re-reads the cache from disk
+    let mut cache = Cache::get_singleton();
+    cache.reload();
+
+    let mut cache_iter = cache.iter();
+
+    loop {
+        let view = match cache_iter.next() {
+            Some(view) => view,
+            None => break
+        };
+
+        let current_version = match view.current_version() {
+            Some(vers) => vers,
+            None => continue
+        };
+        let candidate_version = match view.candidate_version() {
+            Some(vers) => vers,
+            // if there's no candidate (i.e. no update) get info of currently
+            // installed version instead
+            None => current_version.clone()
+        };
+
+        let package = view.name();
+        if filter(&package, &current_version, &candidate_version) {
+            let mut origin_res = "unknown".to_owned();
+            let mut section_res = "unknown".to_owned();
+            let mut priority_res = "unknown".to_owned();
+            let mut change_log_url = "".to_owned();
+            let mut short_desc = package.clone();
+            let mut long_desc = "".to_owned();
+
+            // get additional information via nested APT 'iterators'
+            let mut view_iter = view.versions();
+            while let Some(ver) = view_iter.next() {
+                if ver.version() == candidate_version {
+                    if let Some(section) = ver.section() {
+                        section_res = section;
+                    }
+
+                    if let Some(prio) = ver.priority_type() {
+                        priority_res = prio;
+                    }
+
+                    // assume every package has only one origin file (not
+                    // origin, but origin *file*, for some reason those seem to
+                    // be different concepts in APT)
+                    let mut origin_iter = ver.origin_iter();
+                    let origin = origin_iter.next();
+                    if let Some(origin) = origin {
+
+                        if let Some(sd) = origin.short_desc() {
+                            short_desc = sd;
+                        }
+
+                        if let Some(ld) = origin.long_desc() {
+                            long_desc = ld;
+                        }
+
+                        // the package files appear in priority order, meaning
+                        // the one for the candidate version is first
+                        let mut pkg_iter = origin.file();
+                        let pkg_file = pkg_iter.next();
+                        if let Some(pkg_file) = pkg_file {
+                            if let Some(origin_name) = pkg_file.origin() {
+                                origin_res = origin_name;
+                            }
+
+                            let filename = pkg_file.file_name();
+                            let source_pkg = ver.source_package();
+                            let source_ver = ver.source_version();
+                            let component = pkg_file.component();
+
+                            // build changelog URL from gathered information
+                            // ignore errors, use empty changelog instead
+                            let url = get_changelog_url(&package, &filename, &source_pkg,
+                                &candidate_version, &source_ver, &origin_res, &component);
+                            if let Ok(url) = url {
+                                change_log_url = url;
+                            }
+                        }
+                    }
+
+                    break;
+                }
+            }
+
+            let info = APTUpdateInfo {
+                package,
+                title: short_desc,
+                arch: view.arch(),
+                description: long_desc,
+                change_log_url,
+                origin: origin_res,
+                version: candidate_version,
+                old_version: current_version,
+                priority: priority_res,
+                section: section_res,
+            };
+            ret.push(info);
+        }
+    }
+
+    return ret;
+}
+
+#[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+        },
+    },
+    returns: {
+        description: "A list of packages with available updates.",
+        type: Array,
+        items: { type: APTUpdateInfo },
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// List available APT updates
+fn apt_update_available(_param: Value) -> Result<Value, Error> {
+    let ret = list_installed_apt_packages(|_pkg, cur_ver, can_ver| cur_ver != can_ver);
+    Ok(json!(ret))
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            quiet: {
+                description: "Only produces output suitable for logging, omitting progress indicators.",
+                type: bool,
+                default: false,
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        schema: UPID_SCHEMA,
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Update the APT database
+pub fn apt_update_database(
+    quiet: Option<bool>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
+    let quiet = quiet.unwrap_or(API_METHOD_APT_UPDATE_DATABASE_PARAM_DEFAULT_QUIET);
+
+    let upid_str = WorkerTask::new_thread("aptupdate", None, userid, to_stdout, move |worker| {
+        if !quiet { worker.log("starting apt-get update") }
+
+        // TODO: set proxy /etc/apt/apt.conf.d/76pbsproxy like PVE
+
+        let mut command = std::process::Command::new("apt-get");
+        command.arg("update");
+
+        let output = crate::tools::run_command(command, None)?;
+        if !quiet { worker.log(output) }
+
+        // TODO: add mail notify for new updates like PVE
+
+        Ok(())
+    })?;
+
+    Ok(upid_str)
+}
+
+const SUBDIRS: SubdirMap = &[
+    ("update", &Router::new()
+        .get(&API_METHOD_APT_UPDATE_AVAILABLE)
+        .post(&API_METHOD_APT_UPDATE_DATABASE)
+    ),
+];
+
+pub const ROUTER: Router = Router::new()
+    .get(&list_subdirs_api_method!(SUBDIRS))
+    .subdirs(SUBDIRS);

src/api2/node/disks.rs

@@ -0,0 +1,188 @@
+use anyhow::{bail, Error};
+use serde_json::{json, Value};
+
+use proxmox::api::{api, Permission, RpcEnvironment, RpcEnvironmentType};
+use proxmox::api::router::{Router, SubdirMap};
+use proxmox::{sortable, identity};
+use proxmox::{list_subdirs_api_method};
+
+use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
+use crate::tools::disks::{
+    DiskUsageInfo, DiskUsageType, DiskManage, SmartData,
+    get_disks, get_smart_data, get_disk_usage_info, inititialize_gpt_disk,
+};
+use crate::server::WorkerTask;
+
+use crate::api2::types::{Userid, UPID_SCHEMA, NODE_SCHEMA, BLOCKDEVICE_NAME_SCHEMA};
+
+pub mod directory;
+pub mod zfs;
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            skipsmart: {
+                description: "Skip smart checks.",
+                type: bool,
+                optional: true,
+                default: false,
+            },
+            "usage-type": {
+                type: DiskUsageType,
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        description: "Local disk list.",
+        type: Array,
+        items: {
+            type: DiskUsageInfo,
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "disks"], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// List local disks
+pub fn list_disks(
+    skipsmart: bool,
+    usage_type: Option<DiskUsageType>,
+) -> Result<Vec<DiskUsageInfo>, Error> {
+
+    let mut list = Vec::new();
+
+    for (_, info) in get_disks(None, skipsmart)? {
+        if let Some(ref usage_type) = usage_type {
+            if info.used == *usage_type {
+                list.push(info);
+            }
+        } else {
+            list.push(info);
+        }
+    }
+
+    Ok(list)
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            disk: {
+                schema: BLOCKDEVICE_NAME_SCHEMA,
+            },
+            healthonly: {
+                description: "If true returns only the health status.",
+                type: bool,
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        type: SmartData,
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "disks"], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// Get SMART attributes and health of a disk.
+pub fn smart_status(
+    disk: String,
+    healthonly: Option<bool>,
+) -> Result<SmartData, Error> {
+
+    let healthonly = healthonly.unwrap_or(false);
+
+    let manager = DiskManage::new();
+    let disk = manager.disk_by_name(&disk)?;
+    get_smart_data(&disk, healthonly)
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            disk: {
+                schema: BLOCKDEVICE_NAME_SCHEMA,
+            },
+            uuid: {
+                description: "UUID for the GPT table.",
+                type: String,
+                optional: true,
+                max_length: 36,
+            },
+        },
+    },
+    returns: {
+        schema: UPID_SCHEMA,
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "disks"], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Initialize empty Disk with GPT
+pub fn initialize_disk(
+    disk: String,
+    uuid: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Value, Error> {
+
+    let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
+
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+
+    let info = get_disk_usage_info(&disk, true)?;
+
+    if info.used != DiskUsageType::Unused {
+        bail!("disk '{}' is already in use.", disk);
+    }
+
+    let upid_str = WorkerTask::new_thread(
+        "diskinit", Some(disk.clone()), userid, to_stdout, move |worker|
+        {
+            worker.log(format!("initialize disk {}", disk));
+
+            let disk_manager = DiskManage::new();
+            let disk_info = disk_manager.disk_by_name(&disk)?;
+
+            inititialize_gpt_disk(&disk_info, uuid.as_deref())?;
+
+            Ok(())
+        })?;
+
+    Ok(json!(upid_str))
+}
+
+#[sortable]
+const SUBDIRS: SubdirMap = &sorted!([
+    //    ("lvm", &lvm::ROUTER),
+    ("directory", &directory::ROUTER),
+    ("zfs", &zfs::ROUTER),
+    (
+        "initgpt", &Router::new()
+            .post(&API_METHOD_INITIALIZE_DISK)
+    ),
+    (
+        "list", &Router::new()
+            .get(&API_METHOD_LIST_DISKS)
+    ),
+    (
+        "smart", &Router::new()
+            .get(&API_METHOD_SMART_STATUS)
+    ),
+]);
+
+pub const ROUTER: Router = Router::new()
+    .get(&list_subdirs_api_method!(SUBDIRS))
+    .subdirs(SUBDIRS);

src/api2/node/disks/directory.rs

@@ -0,0 +1,282 @@
+use anyhow::{bail, Error};
+use serde_json::json;
+use ::serde::{Deserialize, Serialize};
+
+use proxmox::api::{api, Permission, RpcEnvironment, RpcEnvironmentType};
+use proxmox::api::section_config::SectionConfigData;
+use proxmox::api::router::Router;
+
+use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
+use crate::tools::disks::{
+    DiskManage, FileSystemType, DiskUsageType,
+    create_file_system, create_single_linux_partition, get_fs_uuid, get_disk_usage_info,
+};
+use crate::tools::systemd::{self, types::*};
+
+use crate::server::WorkerTask;
+
+use crate::api2::types::*;
+use crate::config::datastore::DataStoreConfig;
+
+#[api(
+    properties: {
+        "filesystem": {
+            type: FileSystemType,
+            optional: true,
+        },
+    },
+)]
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all="kebab-case")]
+/// Datastore mount info.
+pub struct DatastoreMountInfo {
+    /// The path of the mount unit.
+    pub unitfile: String,
+    /// The mount path.
+    pub path: String,
+    /// The mounted device.
+    pub device: String,
+    /// File system type
+    pub filesystem: Option<String>,
+    /// Mount options
+    pub options: Option<String>,
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+        }
+    },
+    returns: {
+        description: "List of systemd datastore mount units.",
+        type: Array,
+        items: {
+            type: DatastoreMountInfo,
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "disks"], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// List systemd datastore mount units.
+pub fn  list_datastore_mounts() -> Result<Vec<DatastoreMountInfo>, Error> {
+
+    lazy_static::lazy_static! {
+        static ref MOUNT_NAME_REGEX: regex::Regex = regex::Regex::new(r"^mnt-datastore-(.+)\.mount$").unwrap();
+    }
+
+    let mut list = Vec::new();
+
+    let basedir = "/etc/systemd/system";
+    for item in crate::tools::fs::scan_subdir(libc::AT_FDCWD, basedir, &MOUNT_NAME_REGEX)? {
+        let item = item?;
+        let name = item.file_name().to_string_lossy().to_string();
+
+        let unitfile = format!("{}/{}", basedir, name);
+        let config = systemd::config::parse_systemd_mount(&unitfile)?;
+        let data: SystemdMountSection = config.lookup("Mount", "Mount")?;
+
+        list.push(DatastoreMountInfo {
+            unitfile,
+            device: data.What,
+            path: data.Where,
+            filesystem: data.Type,
+            options: data.Options,
+        });
+    }
+
+    Ok(list)
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            name: {
+                schema: DATASTORE_SCHEMA,
+            },
+            disk: {
+                schema: BLOCKDEVICE_NAME_SCHEMA,
+            },
+            "add-datastore": {
+                description: "Configure a datastore using the directory.",
+                type: bool,
+                optional: true,
+            },
+            filesystem: {
+                type: FileSystemType,
+                optional: true,
+            },
+         }
+    },
+    returns: {
+        schema: UPID_SCHEMA,
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "disks"], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Create a Filesystem on an unused disk. Will be mounted under '/mnt/datastore/<name>'.".
+pub fn create_datastore_disk(
+    name: String,
+    disk: String,
+    add_datastore: Option<bool>,
+    filesystem: Option<FileSystemType>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+
+    let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
+
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+
+    let info = get_disk_usage_info(&disk, true)?;
+
+    if info.used != DiskUsageType::Unused {
+        bail!("disk '{}' is already in use.", disk);
+    }
+
+    let upid_str = WorkerTask::new_thread(
+        "dircreate", Some(name.clone()), userid, to_stdout, move |worker|
+        {
+            worker.log(format!("create datastore '{}' on disk {}", name, disk));
+
+            let add_datastore = add_datastore.unwrap_or(false);
+            let filesystem = filesystem.unwrap_or(FileSystemType::Ext4);
+
+            let manager = DiskManage::new();
+
+            let disk = manager.clone().disk_by_name(&disk)?;
+
+            let partition = create_single_linux_partition(&disk)?;
+            create_file_system(&partition, filesystem)?;
+
+            let uuid = get_fs_uuid(&partition)?;
+            let uuid_path = format!("/dev/disk/by-uuid/{}", uuid);
+
+            let (mount_unit_name, mount_point) = create_datastore_mount_unit(&name, filesystem, &uuid_path)?;
+
+            systemd::reload_daemon()?;
+            systemd::enable_unit(&mount_unit_name)?;
+            systemd::start_unit(&mount_unit_name)?;
+
+            if add_datastore {
+                crate::api2::config::datastore::create_datastore(json!({ "name": name, "path": mount_point }))?
+            }
+
+            Ok(())
+        })?;
+
+    Ok(upid_str)
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            name: {
+                schema: DATASTORE_SCHEMA,
+            },
+        }
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "disks"], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Remove a Filesystem mounted under '/mnt/datastore/<name>'.".
+pub fn delete_datastore_disk(name: String) -> Result<(), Error> {
+
+    let path = format!("/mnt/datastore/{}", name);
+    // path of datastore cannot be changed
+    let (config, _) = crate::config::datastore::config()?;
+    let datastores: Vec<DataStoreConfig> = config.convert_to_typed_array("datastore")?;
+    let conflicting_datastore: Option<DataStoreConfig> = datastores.into_iter()
+        .filter(|ds| ds.path == path)
+        .next();
+
+    if let Some(conflicting_datastore) = conflicting_datastore {
+        bail!("Can't remove '{}' since it's required by datastore '{}'",
+              conflicting_datastore.path, conflicting_datastore.name);
+    }
+
+    // disable systemd mount-unit
+    let mut mount_unit_name = systemd::escape_unit(&path, true);
+    mount_unit_name.push_str(".mount");
+    systemd::disable_unit(&mount_unit_name)?;
+
+    // delete .mount-file
+    let mount_unit_path = format!("/etc/systemd/system/{}", mount_unit_name);
+    let full_path = std::path::Path::new(&mount_unit_path);
+    log::info!("removing systemd mount unit {:?}", full_path);
+    std::fs::remove_file(&full_path)?;
+
+    // try to unmount, if that fails tell the user to reboot or unmount manually
+    let mut command = std::process::Command::new("umount");
+    command.arg(&path);
+    match crate::tools::run_command(command, None) {
+        Err(_) => bail!(
+            "Could not umount '{}' since it is busy. It will stay mounted \
+             until the next reboot or until unmounted manually!",
+            path
+        ),
+        Ok(_) => Ok(())
+    }
+}
+
+const ITEM_ROUTER: Router = Router::new()
+    .delete(&API_METHOD_DELETE_DATASTORE_DISK);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_DATASTORE_MOUNTS)
+    .post(&API_METHOD_CREATE_DATASTORE_DISK)
+    .match_all("name", &ITEM_ROUTER);
+
+
+fn create_datastore_mount_unit(
+    datastore_name: &str,
+    fs_type: FileSystemType,
+    what: &str,
+) -> Result<(String, String), Error> {
+
+    let mount_point = format!("/mnt/datastore/{}", datastore_name);
+    let mut mount_unit_name = systemd::escape_unit(&mount_point, true);
+    mount_unit_name.push_str(".mount");
+
+    let mount_unit_path = format!("/etc/systemd/system/{}", mount_unit_name);
+
+    let unit = SystemdUnitSection {
+        Description: format!("Mount datatstore '{}' under '{}'", datastore_name, mount_point),
+        ..Default::default()
+    };
+
+    let install = SystemdInstallSection {
+        WantedBy: Some(vec!["multi-user.target".to_string()]),
+        ..Default::default()
+    };
+
+    let mount = SystemdMountSection {
+        What: what.to_string(),
+        Where: mount_point.clone(),
+        Type: Some(fs_type.to_string()),
+        Options: Some(String::from("defaults")),
+        ..Default::default()
+    };
+
+    let mut config = SectionConfigData::new();
+    config.set_data("Unit", "Unit", unit)?;
+    config.set_data("Install", "Install", install)?;
+    config.set_data("Mount", "Mount", mount)?;
+
+    systemd::config::save_systemd_mount(&mount_unit_path, &config)?;
+
+    Ok((mount_unit_name, mount_point))
+}

src/api2/node/disks/zfs.rs

@@ -0,0 +1,383 @@
+use anyhow::{bail, Error};
+use serde_json::{json, Value};
+use ::serde::{Deserialize, Serialize};
+
+use proxmox::api::{
+    api, Permission, RpcEnvironment, RpcEnvironmentType,
+    schema::{
+        Schema,
+        StringSchema,
+        ArraySchema,
+        IntegerSchema,
+        ApiStringFormat,
+        parse_property_string,
+    },
+};
+use proxmox::api::router::Router;
+
+use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
+use crate::tools::disks::{
+    zpool_list, zpool_status, parse_zpool_status_config_tree, vdev_list_to_tree,
+    DiskUsageType,
+};
+
+use crate::server::WorkerTask;
+
+use crate::api2::types::*;
+
+pub const DISK_ARRAY_SCHEMA: Schema = ArraySchema::new(
+    "Disk name list.", &BLOCKDEVICE_NAME_SCHEMA)
+    .schema();
+
+pub const DISK_LIST_SCHEMA: Schema = StringSchema::new(
+    "A list of disk names, comma separated.")
+    .format(&ApiStringFormat::PropertyString(&DISK_ARRAY_SCHEMA))
+    .schema();
+
+pub const ZFS_ASHIFT_SCHEMA: Schema = IntegerSchema::new(
+    "Pool sector size exponent.")
+    .minimum(9)
+    .maximum(16)
+    .default(12)
+    .schema();
+
+pub const ZPOOL_NAME_SCHEMA: Schema =StringSchema::new("ZFS Pool Name")
+    .format(&ApiStringFormat::Pattern(&ZPOOL_NAME_REGEX))
+    .schema();
+
+#[api(
+    default: "On",
+)]
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// The ZFS compression algorithm to use.
+pub enum ZfsCompressionType {
+    /// Gnu Zip
+    Gzip,
+    /// LZ4
+    Lz4,
+    /// LZJB
+    Lzjb,
+    /// ZLE
+    Zle,
+    /// Enable compression using the default algorithm.
+    On,
+    /// Disable compression.
+    Off,
+}
+
+#[api()]
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// The ZFS RAID level to use.
+pub enum ZfsRaidLevel {
+    /// Single Disk
+    Single,
+    /// Mirror
+    Mirror,
+    /// Raid10
+    Raid10,
+    /// RaidZ
+    RaidZ,
+    /// RaidZ2
+    RaidZ2,
+    /// RaidZ3
+    RaidZ3,
+}
+
+
+#[api()]
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all="kebab-case")]
+/// zpool list item
+pub struct ZpoolListItem {
+    /// zpool name
+    pub name: String,
+    /// Health
+    pub health: String,
+    /// Total size
+    pub size: u64,
+    /// Used size
+    pub alloc: u64,
+    /// Free space
+    pub free: u64,
+    /// ZFS fragnentation level
+    pub frag: u64,
+    /// ZFS deduplication ratio
+    pub dedup: f64,
+}
+
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+        },
+    },
+    returns: {
+        description: "List of zpools.",
+        type: Array,
+        items: {
+            type: ZpoolListItem,
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "disks"], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// List zfs pools.
+pub fn list_zpools() -> Result<Vec<ZpoolListItem>, Error> {
+
+    let data = zpool_list(None, false)?;
+
+    let mut list = Vec::new();
+
+    for item in data {
+        if let Some(usage) = item.usage {
+            list.push(ZpoolListItem {
+                name: item.name,
+                health: item.health,
+                size: usage.size,
+                alloc: usage.alloc,
+                free: usage.free,
+                frag: usage.frag,
+                dedup: usage.dedup,
+            });
+        }
+    }
+
+    Ok(list)
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            name: {
+                schema: ZPOOL_NAME_SCHEMA,
+            },
+        },
+    },
+    returns: {
+        description: "zpool vdev tree with status",
+        properties: {
+
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "disks"], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// Get zpool status details.
+pub fn zpool_details(
+    name: String,
+) -> Result<Value, Error> {
+
+    let key_value_list = zpool_status(&name)?;
+
+    let config = match key_value_list.iter().find(|(k, _)| k == "config") {
+        Some((_, v)) => v,
+        None =>  bail!("got zpool status without config key"),
+    };
+
+    let vdev_list = parse_zpool_status_config_tree(config)?;
+    let mut tree = vdev_list_to_tree(&vdev_list)?;
+
+    for (k, v) in key_value_list {
+        if k != "config" {
+            tree[k] = v.into();
+        }
+    }
+
+    tree["name"] = tree.as_object_mut().unwrap()
+        .remove("pool")
+        .unwrap_or_else(|| name.into());
+
+
+    Ok(tree)
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+            name: {
+                schema: DATASTORE_SCHEMA,
+            },
+            devices: {
+                schema: DISK_LIST_SCHEMA,
+            },
+            raidlevel: {
+                type: ZfsRaidLevel,
+            },
+            ashift: {
+                schema: ZFS_ASHIFT_SCHEMA,
+                optional: true,
+            },
+            compression: {
+                type: ZfsCompressionType,
+                optional: true,
+            },
+            "add-datastore": {
+                description: "Configure a datastore using the zpool.",
+                type: bool,
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        schema: UPID_SCHEMA,
+    },
+    access: {
+        permission: &Permission::Privilege(&["system", "disks"], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Create a new ZFS pool.
+pub fn create_zpool(
+    name: String,
+    devices: String,
+    raidlevel: ZfsRaidLevel,
+    compression: Option<String>,
+    ashift: Option<usize>,
+    add_datastore: Option<bool>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+
+    let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
+
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+
+    let add_datastore = add_datastore.unwrap_or(false);
+
+    let ashift = ashift.unwrap_or(12);
+
+    let devices_text = devices.clone();
+    let devices = parse_property_string(&devices, &DISK_ARRAY_SCHEMA)?;
+    let devices: Vec<String> = devices.as_array().unwrap().iter()
+        .map(|v| v.as_str().unwrap().to_string()).collect();
+
+    let disk_map = crate::tools::disks::get_disks(None, true)?;
+    for disk in devices.iter() {
+        match disk_map.get(disk) {
+            Some(info) => {
+                if info.used != DiskUsageType::Unused {
+                    bail!("disk '{}' is already in use.", disk);
+                }
+            }
+            None => {
+                bail!("no such disk '{}'", disk);
+            }
+        }
+    }
+
+    let min_disks = match raidlevel {
+        ZfsRaidLevel::Single => 1,
+        ZfsRaidLevel::Mirror => 2,
+        ZfsRaidLevel::Raid10 => 4,
+        ZfsRaidLevel::RaidZ => 3,
+        ZfsRaidLevel::RaidZ2 => 4,
+        ZfsRaidLevel::RaidZ3 => 5,
+    };
+
+    // Sanity checks
+    if raidlevel == ZfsRaidLevel::Raid10 && devices.len() % 2 != 0 {
+        bail!("Raid10 needs an even number of disks.");
+    }
+
+    if raidlevel == ZfsRaidLevel::Single && devices.len() > 1 {
+        bail!("Please give only one disk for single disk mode.");
+    }
+
+    if devices.len() < min_disks {
+        bail!("{:?} needs at least {} disks.", raidlevel, min_disks);
+    }
+
+    // check if the default path does exist already and bail if it does
+    // otherwise we get an error on mounting
+    let mut default_path = std::path::PathBuf::from("/");
+    default_path.push(&name);
+
+    match std::fs::metadata(&default_path) {
+        Err(_) => {}, // path does not exist
+        Ok(_) => {
+            bail!("path {:?} already exists", default_path);
+        }
+    }
+
+     let upid_str = WorkerTask::new_thread(
+        "zfscreate", Some(name.clone()), userid, to_stdout, move |worker|
+        {
+            worker.log(format!("create {:?} zpool '{}' on devices '{}'", raidlevel, name, devices_text));
+
+
+            let mut command = std::process::Command::new("zpool");
+            command.args(&["create", "-o", &format!("ashift={}", ashift), &name]);
+
+            match raidlevel {
+                ZfsRaidLevel::Single => {
+                    command.arg(&devices[0]);
+                }
+                ZfsRaidLevel::Mirror => {
+                    command.arg("mirror");
+                    command.args(devices);
+                }
+                ZfsRaidLevel::Raid10 => {
+                     devices.chunks(2).for_each(|pair| {
+                         command.arg("mirror");
+                         command.args(pair);
+                     });
+                }
+                ZfsRaidLevel::RaidZ => {
+                    command.arg("raidz");
+                    command.args(devices);
+                }
+                ZfsRaidLevel::RaidZ2 => {
+                    command.arg("raidz2");
+                    command.args(devices);
+                }
+                ZfsRaidLevel::RaidZ3 => {
+                    command.arg("raidz3");
+                    command.args(devices);
+                }
+            }
+
+            worker.log(format!("# {:?}", command));
+
+            let output = crate::tools::run_command(command, None)?;
+            worker.log(output);
+
+            if let Some(compression) = compression {
+                let mut command = std::process::Command::new("zfs");
+                command.args(&["set", &format!("compression={}", compression), &name]);
+                worker.log(format!("# {:?}", command));
+                let output = crate::tools::run_command(command, None)?;
+                worker.log(output);
+            }
+
+            if add_datastore {
+                let mount_point = format!("/{}", name);
+                crate::api2::config::datastore::create_datastore(json!({ "name": name, "path": mount_point }))?
+            }
+
+            Ok(())
+        })?;
+
+    Ok(upid_str)
+}
+
+pub const POOL_ROUTER: Router = Router::new()
+    .get(&API_METHOD_ZPOOL_DETAILS);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_ZPOOLS)
+    .post(&API_METHOD_CREATE_ZPOOL)
+    .match_all("name", &POOL_ROUTER);

src/api2/node/journal.rs

@@ -94,7 +94,7 @@ fn get_journal(
 
     let mut lines: Vec<String> = vec![];
 
-    let mut child = Command::new("/usr/bin/mini-journalreader")
+    let mut child = Command::new("mini-journalreader")
         .args(&args)
         .stdout(Stdio::piped())
         .spawn()?;

src/api2/node/network.rs

@@ -4,6 +4,7 @@ use ::serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
 use proxmox::api::schema::parse_property_string;
+use proxmox::tools::fs::open_file_locked;
 
 use crate::config::network::{self, NetworkConfig};
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
@@ -230,7 +231,7 @@ pub fn create_interface(
     let interface_type = crate::tools::required_string_param(&param, "type")?;
     let interface_type: NetworkInterfaceType = serde_json::from_value(interface_type.into())?;
 
-    let _lock = crate::tools::open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let (mut config, _digest) = network::config()?;
 
@@ -338,7 +339,7 @@ pub enum DeletableProperty {
     autostart,
     /// Delete bridge ports (set to 'none')
     bridge_ports,
-    /// Delet bridge-vlan-aware flag
+    /// Delete bridge-vlan-aware flag
     bridge_vlan_aware,
     /// Delete bond-slaves (set to 'none')
     slaves,
@@ -463,7 +464,7 @@ pub fn update_interface(
     param: Value,
 ) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let (mut config, expected_digest) = network::config()?;
 
@@ -586,7 +587,7 @@ pub fn update_interface(
 /// Remove network interface configuration.
 pub fn delete_interface(iface: String, digest: Option<String>) -> Result<(), Error> {
 
-    let _lock = crate::tools::open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0))?;
+    let _lock = open_file_locked(network::NETWORK_LOCKFILE, std::time::Duration::new(10, 0))?;
 
     let (mut config, expected_digest) = network::config()?;
 
@@ -624,9 +625,9 @@ pub async fn reload_network_config(
 
     network::assert_ifupdown2_installed()?;
 
-    let username = rpcenv.get_user().unwrap();
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
 
-    let upid_str = WorkerTask::spawn("srvreload", Some(String::from("networking")), &username.clone(), true, |_worker| async {
+    let upid_str = WorkerTask::spawn("srvreload", Some(String::from("networking")), userid, true, |_worker| async {
 
         let _ = std::fs::rename(network::NETWORK_INTERFACES_NEW_FILENAME, network::NETWORK_INTERFACES_FILENAME);
 

src/api2/node/rrd.rs

@@ -1,9 +1,47 @@
 use anyhow::Error;
-use serde_json::Value;
+use serde_json::{Value, json};
 
 use proxmox::api::{api, Router};
 
 use crate::api2::types::*;
+use crate::tools::epoch_now_f64;
+use crate::rrd::{extract_cached_data, RRD_DATA_ENTRIES};
+
+pub fn create_value_from_rrd(
+    basedir: &str,
+    list: &[&str],
+    timeframe: RRDTimeFrameResolution,
+    cf: RRDMode,
+) -> Result<Value, Error> {
+
+    let mut result = Vec::new();
+    let now = epoch_now_f64()?;
+
+    for name in list {
+        let (start, reso, list) = match extract_cached_data(basedir, name, now, timeframe, cf) {
+            Some(result) => result,
+            None => continue,
+        };
+
+        let mut t = start;
+        for index in 0..RRD_DATA_ENTRIES {
+            if result.len() <= index {
+                if let Some(value) = list[index] {
+                    result.push(json!({ "time": t, *name: value }));
+                } else {
+                    result.push(json!({ "time": t }));
+                }
+            } else {
+                if let Some(value) = list[index] {
+                    result[index][name] = value.into();
+                }
+            }
+            t += reso;
+        }
+    }
+
+    Ok(result.into())
+}
 
 #[api(
     input: {
@@ -27,7 +65,7 @@ fn get_node_stats(
     _param: Value,
 ) -> Result<Value, Error> {
 
-    crate::rrd::extract_data(
+    create_value_from_rrd(
         "host",
         &[
             "cpu", "iowait",
@@ -36,8 +74,9 @@ fn get_node_stats(
             "netin", "netout",
             "loadavg",
             "total", "used",
-            "read_ios", "read_bytes", "read_ticks",
-            "write_ios", "write_bytes", "write_ticks",
+            "read_ios", "read_bytes",
+            "write_ios", "write_bytes",
+            "io_ticks",
          ],
         timeframe,
         cf,

src/api2/node/services.rs

@@ -4,12 +4,13 @@ use anyhow::{bail, Error};
 use serde_json::{json, Value};
 
 use proxmox::{sortable, identity, list_subdirs_api_method};
-use proxmox::api::{api, Router, Permission};
+use proxmox::api::{api, Router, Permission, RpcEnvironment};
 use proxmox::api::router::SubdirMap;
 use proxmox::api::schema::*;
 
 use crate::api2::types::*;
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
+use crate::server::WorkerTask;
 
 static SERVICE_NAME_LIST: [&str; 7] = [
     "proxmox-backup",
@@ -38,7 +39,7 @@ fn get_full_service_state(service: &str) -> Result<Value, Error> {
 
     let real_service_name = real_service_name(service);
 
-    let mut child = Command::new("/bin/systemctl")
+    let mut child = Command::new("systemctl")
         .args(&["show", real_service_name])
         .stdout(Stdio::piped())
         .spawn()?;
@@ -181,30 +182,43 @@ fn get_service_state(
     Ok(json_service_state(&service, status))
 }
 
-fn run_service_command(service: &str, cmd: &str) -> Result<Value, Error> {
+fn run_service_command(service: &str, cmd: &str, userid: Userid) -> Result<Value, Error> {
 
-    // fixme: run background worker (fork_worker) ???
+    let workerid = format!("srv{}", &cmd);
 
-    match cmd {
-        "start"|"stop"|"restart"|"reload" => {},
+    let cmd = match cmd {
+        "start"|"stop"|"restart"=> cmd.to_string(),
+        "reload" => "try-reload-or-restart".to_string(), // some services do not implement reload
         _ => bail!("unknown service command '{}'", cmd),
-    }
+    };
+    let service = service.to_string();
 
-    if service == "proxmox-backup" && cmd != "restart" {
-        bail!("invalid service cmd '{} {}'", service, cmd);
-    }
+    let upid = WorkerTask::new_thread(
+        &workerid,
+        Some(service.clone()),
+        userid,
+        false,
+        move |_worker| {
 
-    let real_service_name = real_service_name(service);
+            if service == "proxmox-backup" && cmd == "stop" {
+                bail!("invalid service cmd '{} {}' cannot stop essential service!", service, cmd);
+            }
 
-    let status = Command::new("/bin/systemctl")
-        .args(&[cmd, real_service_name])
-        .status()?;
+            let real_service_name = real_service_name(&service);
 
-    if !status.success() {
-        bail!("systemctl {} failed with {}", cmd, status);
-    }
+            let status = Command::new("systemctl")
+                .args(&[&cmd, real_service_name])
+                .status()?;
 
-    Ok(Value::Null)
+            if !status.success() {
+                bail!("systemctl {} failed with {}", cmd, status);
+            }
+
+            Ok(())
+        }
+    )?;
+
+    Ok(upid.into())
 }
 
 #[api(
@@ -227,11 +241,14 @@ fn run_service_command(service: &str, cmd: &str) -> Result<Value, Error> {
 fn start_service(
     service: String,
     _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
 
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+
     log::info!("starting service {}", service);
 
-    run_service_command(&service, "start")
+    run_service_command(&service, "start", userid)
 }
 
 #[api(
@@ -254,11 +271,14 @@ fn start_service(
 fn stop_service(
     service: String,
     _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
  ) -> Result<Value, Error> {
 
-    log::info!("stoping service {}", service);
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
 
-    run_service_command(&service, "stop")
+    log::info!("stopping service {}", service);
+
+    run_service_command(&service, "stop", userid)
 }
 
 #[api(
@@ -281,15 +301,18 @@ fn stop_service(
 fn restart_service(
     service: String,
     _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
 
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+
     log::info!("re-starting service {}", service);
 
     if &service == "proxmox-backup-proxy" {
         // special case, avoid aborting running tasks
-        run_service_command(&service, "reload")
+        run_service_command(&service, "reload", userid)
     } else {
-        run_service_command(&service, "restart")
+        run_service_command(&service, "restart", userid)
     }
 }
 
@@ -313,11 +336,14 @@ fn restart_service(
 fn reload_service(
     service: String,
     _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
 
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+
     log::info!("reloading service {}", service);
 
-    run_service_command(&service, "reload")
+    run_service_command(&service, "reload", userid)
 }
 
 

src/api2/node/status.rs

@@ -1,4 +1,5 @@
 use std::process::Command;
+use std::path::Path;
 
 use anyhow::{Error, format_err, bail};
 use serde_json::{json, Value};
@@ -9,6 +10,7 @@ use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission};
 
 use crate::api2::types::*;
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_POWER_MANAGEMENT};
+use crate::tools::cert::CertInfo;
 
 #[api(
     input: {
@@ -45,14 +47,24 @@ use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_POWER_MANAGEMENT};
                 description: "Total CPU usage since last query.",
                 optional: true,
             },
-        }
+            info: {
+                type: Object,
+                description: "contains node information",
+                properties: {
+                    fingerprint: {
+                        description: "The SSL Fingerprint",
+                        type: String,
+                    },
+                },
+            },
+        },
     },
     access: {
         permission: &Permission::Privilege(&["system", "status"], PRIV_SYS_AUDIT, false),
     },
 )]
 /// Read node memory, CPU and (root) disk usage
-fn get_usage(
+fn get_status(
     _param: Value,
     _info: &ApiMethod,
     _rpcenv: &mut dyn RpcEnvironment,
@@ -60,6 +72,11 @@ fn get_usage(
 
     let meminfo: procfs::ProcFsMemInfo = procfs::read_meminfo()?;
     let kstat: procfs::ProcFsStat = procfs::read_proc_stat()?;
+    let disk_usage = crate::tools::disks::disk_usage(Path::new("/"))?;
+
+    // get fingerprint
+    let cert = CertInfo::new()?;
+    let fp = cert.fingerprint()?;
 
     Ok(json!({
         "memory": {
@@ -68,6 +85,14 @@ fn get_usage(
             "free": meminfo.memfree,
         },
         "cpu": kstat.cpu,
+        "root": {
+            "total": disk_usage.total,
+            "used": disk_usage.used,
+            "free": disk_usage.avail,
+        },
+        "info": {
+            "fingerprint": fp,
+        },
     }))
 }
 
@@ -95,7 +120,7 @@ fn reboot_or_shutdown(command: NodePowerCommand) -> Result<(), Error> {
         NodePowerCommand::Shutdown => "poweroff",
     };
 
-    let output = Command::new("/bin/systemctl")
+    let output = Command::new("systemctl")
         .arg(systemctl_command)
         .output()
         .map_err(|err| format_err!("failed to execute systemctl - {}", err))?;
@@ -115,5 +140,5 @@ fn reboot_or_shutdown(command: NodePowerCommand) -> Result<(), Error> {
 }
 
 pub const ROUTER: Router = Router::new()
-    .get(&API_METHOD_GET_USAGE)
+    .get(&API_METHOD_GET_STATUS)
     .post(&API_METHOD_REBOOT_OR_SHUTDOWN);

src/api2/node/subscription.rs

@@ -5,8 +5,16 @@ use proxmox::api::{api, Router, Permission};
 
 use crate::tools;
 use crate::config::acl::PRIV_SYS_AUDIT;
+use crate::api2::types::NODE_SCHEMA;
 
 #[api(
+    input: {
+        properties: {
+            node: {
+                schema: NODE_SCHEMA,
+            },
+        },
+    },
     returns: {
         description: "Subscription status.",
         properties: {

src/api2/node/syslog.rs

@@ -27,7 +27,7 @@ fn dump_journal(
     let start = start.unwrap_or(0);
     let mut count: u64 = 0;
 
-    let mut child = Command::new("/bin/journalctl")
+    let mut child = Command::new("journalctl")
         .args(&args)
         .stdout(Stdio::piped())
         .spawn()?;

src/api2/node/tasks.rs

@@ -4,13 +4,13 @@ use std::io::{BufRead, BufReader};
 use anyhow::{Error};
 use serde_json::{json, Value};
 
-use proxmox::api::{api, Router, RpcEnvironment, Permission, UserInformation};
+use proxmox::api::{api, Router, RpcEnvironment, Permission};
 use proxmox::api::router::SubdirMap;
 use proxmox::{identity, list_subdirs_api_method, sortable};
 
 use crate::tools;
 use crate::api2::types::*;
-use crate::server::{self, UPID};
+use crate::server::{self, UPID, TaskState};
 use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
 use crate::config::cached_user_info::CachedUserInfo;
 
@@ -84,11 +84,11 @@ async fn get_task_status(
 
     let upid = extract_upid(&param)?;
 
-    let username = rpcenv.get_user().unwrap();
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
 
-    if username != upid.username {
+    if userid != upid.userid {
         let user_info = CachedUserInfo::new()?;
-        user_info.check_privs(&username, &["system", "tasks"], PRIV_SYS_AUDIT, false)?;
+        user_info.check_privs(&userid, &["system", "tasks"], PRIV_SYS_AUDIT, false)?;
     }
 
     let mut result = json!({
@@ -99,15 +99,15 @@ async fn get_task_status(
         "starttime": upid.starttime,
         "type": upid.worker_type,
         "id": upid.worker_id,
-        "user": upid.username,
+        "user": upid.userid,
     });
 
     if crate::server::worker_is_active(&upid).await? {
         result["status"] = Value::from("running");
     } else {
-        let exitstatus = crate::server::upid_read_status(&upid).unwrap_or(String::from("unknown"));
+        let exitstatus = crate::server::upid_read_status(&upid).unwrap_or(TaskState::Unknown { endtime: 0 });
         result["status"] = Value::from("stopped");
-        result["exitstatus"] = Value::from(exitstatus);
+        result["exitstatus"] = Value::from(exitstatus.to_string());
     };
 
     Ok(result)
@@ -161,11 +161,11 @@ async fn read_task_log(
 
     let upid = extract_upid(&param)?;
 
-    let username = rpcenv.get_user().unwrap();
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
 
-    if username != upid.username {
+    if userid != upid.userid {
         let user_info = CachedUserInfo::new()?;
-        user_info.check_privs(&username, &["system", "tasks"], PRIV_SYS_AUDIT, false)?;
+        user_info.check_privs(&userid, &["system", "tasks"], PRIV_SYS_AUDIT, false)?;
     }
 
     let test_status = param["test-status"].as_bool().unwrap_or(false);
@@ -234,11 +234,11 @@ fn stop_task(
 
     let upid = extract_upid(&param)?;
 
-    let username = rpcenv.get_user().unwrap();
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
 
-    if username != upid.username {
+    if userid != upid.userid {
         let user_info = CachedUserInfo::new()?;
-        user_info.check_privs(&username, &["system", "tasks"], PRIV_SYS_MODIFY, false)?;
+        user_info.check_privs(&userid, &["system", "tasks"], PRIV_SYS_MODIFY, false)?;
     }
 
     server::abort_worker_async(upid);
@@ -281,7 +281,7 @@ fn stop_task(
                 default: false,
             },
             userfilter: {
-                optional:true,
+                optional: true,
                 type: String,
                 description: "Only list tasks from this user.",
             },
@@ -307,9 +307,9 @@ pub fn list_tasks(
     mut rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Vec<TaskListItem>, Error> {
 
-    let username = rpcenv.get_user().unwrap();
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
     let user_info = CachedUserInfo::new()?;
-    let user_privs = user_info.lookup_privs(&username, &["system", "tasks"]);
+    let user_privs = user_info.lookup_privs(&userid, &["system", "tasks"]);
 
     let list_all = (user_privs & PRIV_SYS_AUDIT) != 0;
 
@@ -323,24 +323,12 @@ pub fn list_tasks(
 
     let mut count = 0;
 
-    for info in list.iter() {
-        if !list_all && info.upid.username != username { continue; }
+    for info in list {
+        if !list_all && info.upid.userid != userid { continue; }
 
-        let mut entry = TaskListItem {
-            upid: info.upid_str.clone(),
-            node: "localhost".to_string(),
-            pid: info.upid.pid as i64,
-            pstart: info.upid.pstart,
-            starttime: info.upid.starttime,
-            worker_type: info.upid.worker_type.clone(),
-            worker_id: info.upid.worker_id.clone(),
-            user: info.upid.username.clone(),
-            endtime: None,
-            status: None,
-        };
 
-        if let Some(username) = userfilter {
-            if !info.upid.username.contains(username) { continue; }
+        if let Some(userid) = userfilter {
+            if !info.upid.userid.as_str().contains(userid) { continue; }
         }
 
         if let Some(store) = store {
@@ -364,12 +352,10 @@ pub fn list_tasks(
 
         if let Some(ref state) = info.state {
             if running { continue; }
-            if errors && state.1 == "OK" {
-                continue;
+            match state {
+                crate::server::TaskState::OK { .. } if errors => continue,
+                _ => {},
             }
-
-            entry.endtime = Some(state.0);
-            entry.status = Some(state.1.clone());
         }
 
         if (count as u64) < start {
@@ -379,7 +365,7 @@ pub fn list_tasks(
             count += 1;
         }
 
-        if (result.len() as u64) < limit { result.push(entry); };
+        if (result.len() as u64) < limit { result.push(info.into()); };
     }
 
     rpcenv["total"] = Value::from(count);

src/api2/pull.rs

@@ -1,6 +1,8 @@
 //! Sync datastore from remote server
+use std::sync::{Arc};
 
 use anyhow::{format_err, Error};
+use futures::{select, future::FutureExt};
 
 use proxmox::api::api;
 use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
@@ -11,10 +13,120 @@ use crate::client::{HttpClient, HttpClientOptions, BackupRepository, pull::pull_
 use crate::api2::types::*;
 use crate::config::{
     remote,
+    sync::SyncJobConfig,
+    jobstate::Job,
     acl::{PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ},
     cached_user_info::CachedUserInfo,
 };
 
+
+pub fn check_pull_privs(
+    userid: &Userid,
+    store: &str,
+    remote: &str,
+    remote_store: &str,
+    delete: bool,
+) -> Result<(), Error> {
+
+    let user_info = CachedUserInfo::new()?;
+
+    user_info.check_privs(userid, &["datastore", store], PRIV_DATASTORE_BACKUP, false)?;
+    user_info.check_privs(userid, &["remote", remote, remote_store], PRIV_REMOTE_READ, false)?;
+
+    if delete {
+        user_info.check_privs(userid, &["datastore", store], PRIV_DATASTORE_PRUNE, false)?;
+    }
+
+    Ok(())
+}
+
+pub async fn get_pull_parameters(
+    store: &str,
+    remote: &str,
+    remote_store: &str,
+) -> Result<(HttpClient, BackupRepository, Arc<DataStore>), Error> {
+
+    let tgt_store = DataStore::lookup_datastore(store)?;
+
+    let (remote_config, _digest) = remote::config()?;
+    let remote: remote::Remote = remote_config.lookup("remote", remote)?;
+
+    let options = HttpClientOptions::new()
+        .password(Some(remote.password.clone()))
+        .fingerprint(remote.fingerprint.clone());
+
+    let client = HttpClient::new(&remote.host, &remote.userid, options)?;
+    let _auth_info = client.login() // make sure we can auth
+        .await
+        .map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?;
+
+    let src_repo = BackupRepository::new(Some(remote.userid), Some(remote.host), remote_store.to_string());
+
+    Ok((client, src_repo, tgt_store))
+}
+
+pub fn do_sync_job(
+    mut job: Job,
+    sync_job: SyncJobConfig,
+    userid: &Userid,
+    schedule: Option<String>,
+) -> Result<String, Error> {
+
+    let job_id = job.jobname().to_string();
+    let worker_type = job.jobtype().to_string();
+
+    let upid_str = WorkerTask::spawn(
+        &worker_type,
+        Some(job.jobname().to_string()),
+        userid.clone(),
+        false,
+        move |worker| async move {
+
+            job.start(&worker.upid().to_string())?;
+
+            let worker2 = worker.clone();
+
+            let worker_future = async move {
+
+                let delete = sync_job.remove_vanished.unwrap_or(true);
+                let (client, src_repo, tgt_store) = get_pull_parameters(&sync_job.store, &sync_job.remote, &sync_job.remote_store).await?;
+
+                worker.log(format!("Starting datastore sync job '{}'", job_id));
+                if let Some(event_str) = schedule {
+                    worker.log(format!("task triggered by schedule '{}'", event_str));
+                }
+                worker.log(format!("Sync datastore '{}' from '{}/{}'",
+                        sync_job.store, sync_job.remote, sync_job.remote_store));
+
+                crate::client::pull::pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, Userid::backup_userid().clone()).await?;
+
+                worker.log(format!("sync job '{}' end", &job_id));
+
+                Ok(())
+            };
+
+            let mut abort_future = worker2.abort_future().map(|_| Err(format_err!("sync aborted")));
+
+            let res = select!{
+                worker = worker_future.fuse() => worker,
+                abort = abort_future => abort,
+            };
+
+            let status = worker2.create_state(&res);
+
+            match job.finish(status) {
+                Ok(_) => {},
+                Err(err) => {
+                    eprintln!("could not finish job state: {}", err);
+                }
+            }
+
+            res
+        })?;
+
+    Ok(upid_str)
+}
+
 #[api(
     input: {
         properties: {
@@ -52,40 +164,19 @@ async fn pull (
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<String, Error> {
 
-    let user_info = CachedUserInfo::new()?;
-
-    let username = rpcenv.get_user().unwrap();
-    user_info.check_privs(&username, &["datastore", &store], PRIV_DATASTORE_BACKUP, false)?;
-    user_info.check_privs(&username, &["remote", &remote, &remote_store], PRIV_REMOTE_READ, false)?;
-
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
     let delete = remove_vanished.unwrap_or(true);
 
-    if delete {
-        user_info.check_privs(&username, &["datastore", &store], PRIV_DATASTORE_PRUNE, false)?;
-    }
+    check_pull_privs(&userid, &store, &remote, &remote_store, delete)?;
 
-    let tgt_store = DataStore::lookup_datastore(&store)?;
-
-    let (remote_config, _digest) = remote::config()?;
-    let remote: remote::Remote = remote_config.lookup("remote", &remote)?;
-
-    let options = HttpClientOptions::new()
-        .password(Some(remote.password.clone()))
-        .fingerprint(remote.fingerprint.clone());
-
-    let client = HttpClient::new(&remote.host, &remote.userid, options)?;
-    let _auth_info = client.login() // make sure we can auth
-        .await
-        .map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?;
-
-    let src_repo = BackupRepository::new(Some(remote.userid), Some(remote.host), remote_store);
+    let (client, src_repo, tgt_store) = get_pull_parameters(&store, &remote, &remote_store).await?;
 
     // fixme: set to_stdout to false?
-    let upid_str = WorkerTask::spawn("sync", Some(store.clone()), &username.clone(), true, move |worker| async move {
+    let upid_str = WorkerTask::spawn("sync", Some(store.clone()), userid.clone(), true, move |worker| async move {
 
         worker.log(format!("sync datastore '{}' start", store));
 
-        pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, username).await?;
+        pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, userid).await?;
 
         worker.log(format!("sync datastore '{}' end", store));
 

src/api2/reader.rs

@@ -17,6 +17,7 @@ use crate::server::{WorkerTask, H2Service};
 use crate::tools;
 use crate::config::acl::PRIV_DATASTORE_READ;
 use crate::config::cached_user_info::CachedUserInfo;
+use crate::api2::helpers;
 
 mod environment;
 use environment::*;
@@ -54,11 +55,11 @@ fn upgrade_to_backup_reader_protocol(
     async move {
         let debug = param["debug"].as_bool().unwrap_or(false);
 
-        let username = rpcenv.get_user().unwrap();
+        let userid: Userid = rpcenv.get_user().unwrap().parse()?;
         let store = tools::required_string_param(&param, "store")?.to_owned();
 
         let user_info = CachedUserInfo::new()?;
-        user_info.check_privs(&username, &["datastore", &store], PRIV_DATASTORE_READ, false)?;
+        user_info.check_privs(&userid, &["datastore", &store], PRIV_DATASTORE_READ, false)?;
 
         let datastore = DataStore::lookup_datastore(&store)?;
 
@@ -89,9 +90,14 @@ fn upgrade_to_backup_reader_protocol(
 
         let worker_id = format!("{}_{}_{}_{:08X}", store, backup_type, backup_id, backup_dir.backup_time().timestamp());
 
-        WorkerTask::spawn("reader", Some(worker_id), &username.clone(), true, move |worker| {
+        WorkerTask::spawn("reader", Some(worker_id), userid.clone(), true, move |worker| {
             let mut env = ReaderEnvironment::new(
-                env_type, username.clone(), worker.clone(), datastore, backup_dir);
+                env_type,
+                userid,
+                worker.clone(),
+                datastore,
+                backup_dir,
+            );
 
             env.debug = debug;
 
@@ -131,7 +137,7 @@ fn upgrade_to_backup_reader_protocol(
                     Either::Right((Ok(res), _)) => Ok(res),
                     Either::Right((Err(err), _)) => Err(err),
                 })
-                .map_ok(move |_| env.log("reader finished sucessfully"))
+                .map_ok(move |_| env.log("reader finished successfully"))
         })?;
 
         let response = Response::builder()
@@ -187,26 +193,9 @@ fn download_file(
         path.push(env.backup_dir.relative_path());
         path.push(&file_name);
 
-        let path2 = path.clone();
-        let path3 = path.clone();
+        env.log(format!("download {:?}", path.clone()));
 
-        let file = tokio::fs::File::open(path)
-            .map_err(move |err| http_err!(BAD_REQUEST, format!("open file {:?} failed: {}", path2, err)))
-            .await?;
-
-        env.log(format!("download {:?}", path3));
-
-        let payload = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
-            .map_ok(|bytes| hyper::body::Bytes::from(bytes.freeze()));
-
-        let body = Body::wrap_stream(payload);
-
-        // fixme: set other headers ?
-        Ok(Response::builder()
-           .status(StatusCode::OK)
-           .header(header::CONTENT_TYPE, "application/octet-stream")
-           .body(body)
-           .unwrap())
+        helpers::create_download_response(path).await
     }.boxed()
 }
 
@@ -241,8 +230,8 @@ fn download_chunk(
         env.debug(format!("download chunk {:?}", path));
 
         let data = tokio::fs::read(path)
-            .map_err(move |err| http_err!(BAD_REQUEST, format!("reading file {:?} failed: {}", path2, err)))
-            .await?;
+            .await
+            .map_err(move |err| http_err!(BAD_REQUEST, "reading file {:?} failed: {}", path2, err))?;
 
         let body = Body::from(data);
 
@@ -276,7 +265,7 @@ fn download_chunk_old(
     let path3 = path.clone();
 
     let response_future = tokio::fs::File::open(path)
-        .map_err(move |err| http_err!(BAD_REQUEST, format!("open file {:?} failed: {}", path2, err)))
+        .map_err(move |err| http_err!(BAD_REQUEST, "open file {:?} failed: {}", path2, err))
         .and_then(move |file| {
             env2.debug(format!("download chunk {:?}", path3));
             let payload = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())

src/api2/reader/environment.rs

@@ -5,9 +5,10 @@ use serde_json::{json, Value};
 
 use proxmox::api::{RpcEnvironment, RpcEnvironmentType};
 
-use crate::server::WorkerTask;
+use crate::api2::types::Userid;
 use crate::backup::*;
 use crate::server::formatter::*;
+use crate::server::WorkerTask;
 
 //use proxmox::tools;
 
@@ -16,7 +17,7 @@ use crate::server::formatter::*;
 pub struct ReaderEnvironment {
     env_type: RpcEnvironmentType,
     result_attributes: Value,
-    user: String,
+    user: Userid,
     pub debug: bool,
     pub formatter: &'static OutputFormatter,
     pub worker: Arc<WorkerTask>,
@@ -28,7 +29,7 @@ pub struct ReaderEnvironment {
 impl ReaderEnvironment {
     pub fn new(
         env_type: RpcEnvironmentType,
-        user: String,
+        user: Userid,
         worker: Arc<WorkerTask>,
         datastore: Arc<DataStore>,
         backup_dir: BackupDir,
@@ -77,7 +78,7 @@ impl RpcEnvironment for ReaderEnvironment {
     }
 
     fn get_user(&self) -> Option<String> {
-        Some(self.user.clone())
+        Some(self.user.to_string())
     }
 }
 

src/api2/status.rs

@@ -0,0 +1,228 @@
+use proxmox::list_subdirs_api_method;
+
+use anyhow::{Error};
+use serde_json::{json, Value};
+
+use proxmox::api::{
+    api,
+    ApiMethod,
+    Permission,
+    Router,
+    RpcEnvironment,
+    SubdirMap,
+};
+
+use crate::api2::types::{
+    DATASTORE_SCHEMA,
+    RRDMode,
+    RRDTimeFrameResolution,
+    TaskListItem,
+    Userid,
+};
+
+use crate::server;
+use crate::backup::{DataStore};
+use crate::config::datastore;
+use crate::tools::epoch_now_f64;
+use crate::tools::statistics::{linear_regression};
+use crate::config::cached_user_info::CachedUserInfo;
+use crate::config::acl::{
+    PRIV_SYS_AUDIT,
+    PRIV_DATASTORE_AUDIT,
+    PRIV_DATASTORE_BACKUP,
+};
+
+#[api(
+    returns: {
+        description: "Lists the Status of the Datastores.",
+        type: Array,
+        items: {
+            description: "Status of a Datastore",
+            type: Object,
+            properties: {
+                store: {
+                    schema: DATASTORE_SCHEMA,
+                },
+                total: {
+                    type: Integer,
+                    description: "The Size of the underlying storage in bytes",
+                },
+                used: {
+                    type: Integer,
+                    description: "The used bytes of the underlying storage",
+                },
+                avail: {
+                    type: Integer,
+                    description: "The available bytes of the underlying storage",
+                },
+                history: {
+                    type: Array,
+                    description: "A list of usages of the past (last Month).",
+                    items: {
+                        type: Number,
+                        description: "The usage of a time in the past. Either null or between 0.0 and 1.0.",
+                    }
+                },
+                "estimated-full-date": {
+                    type: Integer,
+                    optional: true,
+                    description: "Estimation of the UNIX epoch when the storage will be full.\
+                        This is calculated via a simple Linear Regression (Least Squares)\
+                        of RRD data of the last Month. Missing if there are not enough data points yet.\
+                        If the estimate lies in the past, the usage is decreasing.",
+                },
+            },
+        },
+    },
+)]
+/// List Datastore usages and estimates
+fn datastore_status(
+    _param: Value,
+    _info: &ApiMethod,
+    rpcenv: &mut dyn RpcEnvironment,
+    ) -> Result<Value, Error> {
+
+    let (config, _digest) = datastore::config()?;
+
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let mut list = Vec::new();
+
+    for (store, (_, _)) in &config.sections {
+        let user_privs = user_info.lookup_privs(&userid, &["datastore", &store]);
+        let allowed = (user_privs & (PRIV_DATASTORE_AUDIT| PRIV_DATASTORE_BACKUP)) != 0;
+        if !allowed {
+            continue;
+        }
+
+        let datastore = DataStore::lookup_datastore(&store)?;
+        let status = crate::tools::disks::disk_usage(&datastore.base_path())?;
+
+        let mut entry = json!({
+            "store": store,
+            "total": status.total,
+            "used": status.used,
+            "avail": status.avail,
+        });
+
+        let rrd_dir = format!("datastore/{}", store);
+        let now = epoch_now_f64()?;
+        let rrd_resolution = RRDTimeFrameResolution::Month;
+        let rrd_mode = RRDMode::Average;
+
+        let total_res = crate::rrd::extract_cached_data(
+            &rrd_dir,
+            "total",
+            now,
+            rrd_resolution,
+            rrd_mode,
+        );
+
+        let used_res = crate::rrd::extract_cached_data(
+            &rrd_dir,
+            "used",
+            now,
+            rrd_resolution,
+            rrd_mode,
+        );
+
+        match (total_res, used_res) {
+            (Some((start, reso, total_list)), Some((_, _, used_list))) => {
+                let mut usage_list: Vec<f64> = Vec::new();
+                let mut time_list: Vec<u64> = Vec::new();
+                let mut history = Vec::new();
+
+                for (idx, used) in used_list.iter().enumerate() {
+                    let total = if idx < total_list.len() {
+                        total_list[idx]
+                    } else {
+                        None
+                    };
+
+                    match (total, used) {
+                        (Some(total), Some(used)) if total != 0.0 => {
+                            time_list.push(start + (idx as u64)*reso);
+                            let usage = used/total;
+                            usage_list.push(usage);
+                            history.push(json!(usage));
+                        },
+                        _ => {
+                            history.push(json!(null))
+                        }
+                    }
+                }
+
+                entry["history"] = history.into();
+
+                // we skip the calculation for datastores with not enough data
+                if usage_list.len() >= 7 {
+                    if let Some((a,b)) = linear_regression(&time_list, &usage_list) {
+                        if b != 0.0 {
+                            let estimate = (1.0 - a) / b;
+                            entry["estimated-full-date"] = Value::from(estimate.floor() as u64);
+                        } else {
+                            entry["estimated-full-date"] = Value::from(0);
+                        }
+                    }
+                }
+            },
+            _ => {},
+        }
+
+        list.push(entry);
+    }
+
+    Ok(list.into())
+}
+
+#[api(
+    input: {
+        properties: {
+            since: {
+                type: u64,
+                description: "Only list tasks since this UNIX epoch.",
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        description: "A list of tasks.",
+        type: Array,
+        items: { type: TaskListItem },
+    },
+    access: {
+        description: "Users can only see there own tasks, unless the have Sys.Audit on /system/tasks.",
+        permission: &Permission::Anybody,
+    },
+)]
+/// List tasks.
+pub fn list_tasks(
+    _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<TaskListItem>, Error> {
+
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    let user_privs = user_info.lookup_privs(&userid, &["system", "tasks"]);
+
+    let list_all = (user_privs & PRIV_SYS_AUDIT) != 0;
+
+    // TODO: replace with call that gets all task since 'since' epoch
+    let list: Vec<TaskListItem> = server::read_task_list()?
+        .into_iter()
+        .map(TaskListItem::from)
+        .filter(|entry| list_all || entry.user == userid)
+        .collect();
+
+    Ok(list.into())
+}
+
+const SUBDIRS: SubdirMap = &[
+    ("datastore-usage", &Router::new().get(&API_METHOD_DATASTORE_STATUS)),
+    ("tasks", &Router::new().get(&API_METHOD_LIST_TASKS)),
+];
+
+pub const ROUTER: Router = Router::new()
+    .get(&list_subdirs_api_method!(SUBDIRS))
+    .subdirs(SUBDIRS);

src/api2/types/macros.rs

@@ -0,0 +1,4 @@
+//! Macros exported from api2::types.
+
+#[macro_export]
+macro_rules! PROXMOX_SAFE_ID_REGEX_STR {  () => (r"(?:[A-Za-z0-9_][A-Za-z0-9._\-]*)") }

src/api2/types/mod.rs

@@ -1,10 +1,23 @@
-use anyhow::{bail};
-use ::serde::{Deserialize, Serialize};
+use anyhow::bail;
+use serde::{Deserialize, Serialize};
 
 use proxmox::api::{api, schema::*};
 use proxmox::const_regex;
 use proxmox::{IPRE, IPV4RE, IPV6RE, IPV4OCTET, IPV6H16, IPV6LS32};
 
+use crate::backup::CryptMode;
+use crate::server::UPID;
+
+#[macro_use]
+mod macros;
+
+#[macro_use]
+mod userid;
+pub use userid::{Realm, RealmRef};
+pub use userid::{Username, UsernameRef};
+pub use userid::Userid;
+pub use userid::PROXMOX_GROUP_ID_SCHEMA;
+
 // File names: may not contain slashes, may not start with "."
 pub const FILENAME_FORMAT: ApiStringFormat = ApiStringFormat::VerifyFn(|name| {
     if name.starts_with('.') {
@@ -19,17 +32,6 @@ pub const FILENAME_FORMAT: ApiStringFormat = ApiStringFormat::VerifyFn(|name| {
 macro_rules! DNS_LABEL { () => (r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]*[a-zA-Z0-9])?)") }
 macro_rules! DNS_NAME { () => (concat!(r"(?:", DNS_LABEL!() , r"\.)*", DNS_LABEL!())) }
 
-// we only allow a limited set of characters
-// colon is not allowed, because we store usernames in
-// colon separated lists)!
-// slash is not allowed because it is used as pve API delimiter
-// also see "man useradd"
-macro_rules! USER_NAME_REGEX_STR { () => (r"(?:[^\s:/[:cntrl:]]+)") }
-macro_rules! GROUP_NAME_REGEX_STR { () => (USER_NAME_REGEX_STR!()) }
-
-#[macro_export]
-macro_rules! PROXMOX_SAFE_ID_REGEX_STR {  () => (r"(?:[A-Za-z0-9_][A-Za-z0-9._\-]*)") }
-
 macro_rules! CIDR_V4_REGEX_STR { () => (concat!(r"(?:", IPV4RE!(), r"/\d{1,2})$")) }
 macro_rules! CIDR_V6_REGEX_STR { () => (concat!(r"(?:", IPV6RE!(), r"/\d{1,3})$")) }
 
@@ -63,13 +65,15 @@ const_regex!{
 
     pub DNS_NAME_OR_IP_REGEX = concat!(r"^", DNS_NAME!(), "|",  IPRE!(), r"$");
 
-    pub PROXMOX_USER_ID_REGEX = concat!(r"^",  USER_NAME_REGEX_STR!(), r"@", PROXMOX_SAFE_ID_REGEX_STR!(), r"$");
-
-    pub PROXMOX_GROUP_ID_REGEX = concat!(r"^",  GROUP_NAME_REGEX_STR!(), r"$");
+    pub BACKUP_REPO_URL_REGEX = concat!(r"^^(?:(?:(", USER_ID_REGEX_STR!(), ")@)?(", DNS_NAME!(), "|",  IPRE!() ,"):)?(", PROXMOX_SAFE_ID_REGEX_STR!(), r")$");
 
     pub CERT_FINGERPRINT_SHA256_REGEX = r"^(?:[0-9a-fA-F][0-9a-fA-F])(?::[0-9a-fA-F][0-9a-fA-F]){31}$";
 
     pub ACL_PATH_REGEX = concat!(r"^(?:/|", r"(?:/", PROXMOX_SAFE_ID_REGEX_STR!(), ")+", r")$");
+
+    pub BLOCKDEVICE_NAME_REGEX = r"^(:?(:?h|s|x?v)d[a-z]+)|(:?nvme\d+n\d+)$";
+
+    pub ZPOOL_NAME_REGEX = r"^[a-zA-Z][a-z0-9A-Z\-_.:]+$";
 }
 
 pub const SYSTEMD_DATETIME_FORMAT: ApiStringFormat =
@@ -105,12 +109,6 @@ pub const DNS_NAME_FORMAT: ApiStringFormat =
 pub const DNS_NAME_OR_IP_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&DNS_NAME_OR_IP_REGEX);
 
-pub const PROXMOX_USER_ID_FORMAT: ApiStringFormat =
-    ApiStringFormat::Pattern(&PROXMOX_USER_ID_REGEX);
-
-pub const PROXMOX_GROUP_ID_FORMAT: ApiStringFormat =
-    ApiStringFormat::Pattern(&PROXMOX_GROUP_ID_REGEX);
-
 pub const PASSWORD_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&PASSWORD_REGEX);
 
@@ -129,6 +127,8 @@ pub const CIDR_V6_FORMAT: ApiStringFormat =
 pub const CIDR_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&CIDR_REGEX);
 
+pub const BLOCKDEVICE_NAME_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&BLOCKDEVICE_NAME_REGEX);
 
 pub const PASSWORD_SCHEMA: Schema = StringSchema::new("Password.")
     .format(&PASSWORD_FORMAT)
@@ -287,6 +287,11 @@ pub const DATASTORE_SCHEMA: Schema = StringSchema::new("Datastore name.")
     .max_length(32)
     .schema();
 
+pub const SYNC_SCHEDULE_SCHEMA: Schema = StringSchema::new(
+    "Run sync job at specified schedule.")
+    .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
+    .schema();
+
 pub const GC_SCHEDULE_SCHEMA: Schema = StringSchema::new(
     "Run garbage collection job at specified schedule.")
     .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
@@ -326,25 +331,12 @@ pub const DNS_NAME_OR_IP_SCHEMA: Schema = StringSchema::new("DNS name or IP addr
     .format(&DNS_NAME_OR_IP_FORMAT)
     .schema();
 
-pub const PROXMOX_AUTH_REALM_SCHEMA: Schema = StringSchema::new("Authentication domain ID")
-    .format(&PROXMOX_SAFE_ID_FORMAT)
-    .min_length(3)
-    .max_length(32)
-    .schema();
-
-pub const PROXMOX_USER_ID_SCHEMA: Schema = StringSchema::new("User ID")
-    .format(&PROXMOX_USER_ID_FORMAT)
+pub const BLOCKDEVICE_NAME_SCHEMA: Schema = StringSchema::new("Block device name (/sys/block/<name>).")
+    .format(&BLOCKDEVICE_NAME_FORMAT)
     .min_length(3)
     .max_length(64)
     .schema();
 
-pub const PROXMOX_GROUP_ID_SCHEMA: Schema = StringSchema::new("Group ID")
-    .format(&PROXMOX_GROUP_ID_FORMAT)
-    .min_length(3)
-    .max_length(64)
-    .schema();
-
-
 // Complex type definitions
 
 #[api(
@@ -366,6 +358,10 @@ pub const PROXMOX_GROUP_ID_SCHEMA: Schema = StringSchema::new("Group ID")
                 schema: BACKUP_ARCHIVE_NAME_SCHEMA
             },
         },
+        owner: {
+            type: Userid,
+            optional: true,
+        },
     },
 )]
 #[derive(Serialize, Deserialize)]
@@ -379,6 +375,28 @@ pub struct GroupListItem {
     pub backup_count: u64,
     /// List of contained archive files.
     pub files: Vec<String>,
+    /// The owner of group
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub owner: Option<Userid>,
+}
+
+#[api(
+    properties: {
+        upid: {
+            schema: UPID_SCHEMA
+        },
+        state: {
+            type: String
+        },
+    },
+)]
+#[derive(Serialize, Deserialize)]
+/// Task properties.
+pub struct SnapshotVerifyState {
+    /// UPID of the verify task
+    pub upid: UPID,
+    /// State of the verification. "failed" or "ok"
+    pub state: String,
 }
 
 #[api(
@@ -392,11 +410,23 @@ pub struct GroupListItem {
         "backup-time": {
             schema: BACKUP_TIME_SCHEMA,
         },
+        comment: {
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+            optional: true,
+        },
+        verification: {
+            type: SnapshotVerifyState,
+            optional: true,
+        },
         files: {
             items: {
                 schema: BACKUP_ARCHIVE_NAME_SCHEMA
             },
         },
+        owner: {
+            type: Userid,
+            optional: true,
+        },
     },
 )]
 #[derive(Serialize, Deserialize)]
@@ -406,11 +436,20 @@ pub struct SnapshotListItem {
     pub backup_type: String, // enum
     pub backup_id: String,
     pub backup_time: i64,
+    /// The first line from manifest "notes"
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub comment: Option<String>,
+    /// The result of the last run verify task
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub verification: Option<SnapshotVerifyState>,
     /// List of contained archive files.
-    pub files: Vec<String>,
+    pub files: Vec<BackupContent>,
     /// Overall snapshot size (sum of all archive sizes).
     #[serde(skip_serializing_if="Option::is_none")]
     pub size: Option<u64>,
+    /// The owner of the snapshots group
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub owner: Option<Userid>,
 }
 
 #[api(
@@ -472,6 +511,10 @@ pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema = IntegerSchema::new(
         "filename": {
             schema: BACKUP_ARCHIVE_NAME_SCHEMA,
         },
+        "crypt-mode": {
+            type: CryptMode,
+            optional: true,
+        },
     },
 )]
 #[derive(Serialize, Deserialize)]
@@ -479,6 +522,9 @@ pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema = IntegerSchema::new(
 /// Basic information about archive files inside a backup snapshot.
 pub struct BackupContent {
     pub filename: String,
+    /// Info if file is encrypted, signed, or neither.
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub crypt_mode: Option<CryptMode>,
     /// Archive size (from backup manifest).
     #[serde(skip_serializing_if="Option::is_none")]
     pub size: Option<u64>,
@@ -546,7 +592,8 @@ pub struct StorageStatus {
 
 #[api(
     properties: {
-        "upid": { schema: UPID_SCHEMA },
+        upid: { schema: UPID_SCHEMA },
+        user: { type: Userid },
     },
 )]
 #[derive(Serialize, Deserialize)]
@@ -566,7 +613,7 @@ pub struct TaskListItem {
     /// Worker ID (arbitrary ASCII string)
     pub worker_id: Option<String>,
     /// The user who started the task
-    pub user: String,
+    pub user: Userid,
     /// The task end time (Epoch)
     #[serde(skip_serializing_if="Option::is_none")]
     pub endtime: Option<i64>,
@@ -575,6 +622,27 @@ pub struct TaskListItem {
     pub status: Option<String>,
 }
 
+impl From<crate::server::TaskListInfo> for TaskListItem {
+    fn from(info: crate::server::TaskListInfo) -> Self {
+        let (endtime, status) = info
+            .state
+            .map_or_else(|| (None, None), |a| (Some(a.endtime()), Some(a.to_string())));
+
+        TaskListItem {
+            upid: info.upid_str,
+            node: "localhost".to_string(),
+            pid: info.upid.pid as i64,
+            pstart: info.upid.pstart,
+            starttime: info.upid.starttime,
+            worker_type: info.upid.worker_type,
+            worker_id: info.upid.worker_id,
+            user: info.upid.userid,
+            endtime,
+            status,
+        }
+    }
+}
+
 #[api()]
 #[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "lowercase")]
@@ -807,7 +875,7 @@ fn test_cert_fingerprint_schema() -> Result<(), anyhow::Error> {
 
     for fingerprint in invalid_fingerprints.iter() {
         if let Ok(_) = parse_simple_value(fingerprint, &schema) {
-            bail!("test fingerprint '{}' failed -  got Ok() while expection an error.", fingerprint);
+            bail!("test fingerprint '{}' failed -  got Ok() while exception an error.", fingerprint);
         }
     }
 
@@ -834,9 +902,6 @@ fn test_cert_fingerprint_schema() -> Result<(), anyhow::Error> {
 
 #[test]
 fn test_proxmox_user_id_schema() -> Result<(), anyhow::Error> {
-
-    let schema = PROXMOX_USER_ID_SCHEMA;
-
     let invalid_user_ids = [
         "x", // too short
         "xx", // too short
@@ -850,8 +915,8 @@ fn test_proxmox_user_id_schema() -> Result<(), anyhow::Error> {
     ];
 
     for name in invalid_user_ids.iter() {
-        if let Ok(_) = parse_simple_value(name, &schema) {
-            bail!("test userid '{}' failed -  got Ok() while expection an error.", name);
+        if let Ok(_) = parse_simple_value(name, &Userid::API_SCHEMA) {
+            bail!("test userid '{}' failed -  got Ok() while exception an error.", name);
         }
     }
 
@@ -864,7 +929,7 @@ fn test_proxmox_user_id_schema() -> Result<(), anyhow::Error> {
     ];
 
     for name in valid_user_ids.iter() {
-        let v = match parse_simple_value(name, &schema) {
+        let v = match parse_simple_value(name, &Userid::API_SCHEMA) {
             Ok(v) => v,
             Err(err) => {
                 bail!("unable to parse userid '{}' - {}", name, err);
@@ -906,3 +971,30 @@ pub enum RRDTimeFrameResolution {
     /// 1 week => last 490 days
     Year = 60*10080,
 }
+
+#[api()]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+/// Describes a package for which an update is available.
+pub struct APTUpdateInfo {
+    /// Package name
+    pub package: String,
+    /// Package title
+    pub title: String,
+    /// Package architecture
+    pub arch: String,
+    /// Human readable package description
+    pub description: String,
+    /// New version to be updated to
+    pub version: String,
+    /// Old version currently installed
+    pub old_version: String,
+    /// Package origin
+    pub origin: String,
+    /// Package priority in human-readable form
+    pub priority: String,
+    /// Package section
+    pub section: String,
+    /// URL under which the package's changelog can be retrieved
+    pub change_log_url: String,
+}

src/api2/types/userid.rs

@@ -0,0 +1,420 @@
+//! Types for user handling.
+//!
+//! We have [`Username`]s and [`Realm`]s. To uniquely identify a user, they must be combined into a [`Userid`].
+//!
+//! Since they're all string types, they're organized as follows:
+//!
+//! * [`Username`]: an owned user name. Internally a `String`.
+//! * [`UsernameRef`]: a borrowed user name. Pairs with a `Username` the same way a `str` pairs
+//!   with `String`, meaning you can only make references to it.
+//! * [`Realm`]: an owned realm (`String` equivalent).
+//! * [`RealmRef`]: a borrowed realm (`str` equivalent).
+//! * [`Userid`]: an owned user id (`"user@realm"`). Note that this does not have a separate
+//!   borrowed type.
+//!
+//! Note that `Username`s are not unique, therefore they do not implement `Eq` and cannot be
+//! compared directly. If a direct comparison is really required, they can be compared as strings
+//! via the `as_str()` method. [`Realm`]s and [`Userid`]s on the other hand can be compared with
+//! each other, as in those two cases the comparison has meaning.
+
+use std::borrow::Borrow;
+use std::convert::TryFrom;
+use std::fmt;
+
+use anyhow::{bail, format_err, Error};
+use lazy_static::lazy_static;
+use serde::{Deserialize, Serialize};
+
+use proxmox::api::api;
+use proxmox::api::schema::{ApiStringFormat, Schema, StringSchema};
+use proxmox::const_regex;
+
+// we only allow a limited set of characters
+// colon is not allowed, because we store usernames in
+// colon separated lists)!
+// slash is not allowed because it is used as pve API delimiter
+// also see "man useradd"
+macro_rules! USER_NAME_REGEX_STR { () => (r"(?:[^\s:/[:cntrl:]]+)") }
+macro_rules! GROUP_NAME_REGEX_STR { () => (USER_NAME_REGEX_STR!()) }
+macro_rules! USER_ID_REGEX_STR { () => (concat!(USER_NAME_REGEX_STR!(), r"@", PROXMOX_SAFE_ID_REGEX_STR!())) }
+
+const_regex! {
+    pub PROXMOX_USER_NAME_REGEX = concat!(r"^",  USER_NAME_REGEX_STR!(), r"$");
+    pub PROXMOX_USER_ID_REGEX = concat!(r"^",  USER_ID_REGEX_STR!(), r"$");
+    pub PROXMOX_GROUP_ID_REGEX = concat!(r"^",  GROUP_NAME_REGEX_STR!(), r"$");
+}
+
+pub const PROXMOX_USER_NAME_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&PROXMOX_USER_NAME_REGEX);
+
+pub const PROXMOX_USER_ID_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&PROXMOX_USER_ID_REGEX);
+
+pub const PROXMOX_GROUP_ID_FORMAT: ApiStringFormat =
+    ApiStringFormat::Pattern(&PROXMOX_GROUP_ID_REGEX);
+
+pub const PROXMOX_GROUP_ID_SCHEMA: Schema = StringSchema::new("Group ID")
+    .format(&PROXMOX_GROUP_ID_FORMAT)
+    .min_length(3)
+    .max_length(64)
+    .schema();
+
+pub const PROXMOX_AUTH_REALM_STRING_SCHEMA: StringSchema =
+    StringSchema::new("Authentication domain ID")
+        .format(&super::PROXMOX_SAFE_ID_FORMAT)
+        .min_length(3)
+        .max_length(32);
+pub const PROXMOX_AUTH_REALM_SCHEMA: Schema = PROXMOX_AUTH_REALM_STRING_SCHEMA.schema();
+
+
+#[api(
+    type: String,
+    format: &PROXMOX_USER_NAME_FORMAT,
+)]
+/// The user name part of a user id.
+///
+/// This alone does NOT uniquely identify the user and therefore does not implement `Eq`. In order
+/// to compare user names directly, they need to be explicitly compared as strings by calling
+/// `.as_str()`.
+///
+/// ```compile_fail
+/// fn test(a: Username, b: Username) -> bool {
+///     a == b // illegal and does not compile
+/// }
+/// ```
+#[derive(Clone, Debug, Hash, Deserialize, Serialize)]
+pub struct Username(String);
+
+/// A reference to a user name part of a user id. This alone does NOT uniquely identify the user.
+///
+/// This is like a `str` to the `String` of a [`Username`].
+#[derive(Debug, Hash)]
+pub struct UsernameRef(str);
+
+#[doc(hidden)]
+/// ```compile_fail
+/// let a: Username = unsafe { std::mem::zeroed() };
+/// let b: Username = unsafe { std::mem::zeroed() };
+/// let _ = <Username as PartialEq>::eq(&a, &b);
+/// ```
+///
+/// ```compile_fail
+/// let a: &UsernameRef = unsafe { std::mem::zeroed() };
+/// let b: &UsernameRef = unsafe { std::mem::zeroed() };
+/// let _ = <&UsernameRef as PartialEq>::eq(a, b);
+/// ```
+///
+/// ```compile_fail
+/// let a: &UsernameRef = unsafe { std::mem::zeroed() };
+/// let b: &UsernameRef = unsafe { std::mem::zeroed() };
+/// let _ = <&UsernameRef as PartialEq>::eq(&a, &b);
+/// ```
+struct _AssertNoEqImpl;
+
+impl UsernameRef {
+    fn new(s: &str) -> &Self {
+        unsafe { &*(s as *const str as *const UsernameRef) }
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl std::ops::Deref for Username {
+    type Target = UsernameRef;
+
+    fn deref(&self) -> &UsernameRef {
+        self.borrow()
+    }
+}
+
+impl Borrow<UsernameRef> for Username {
+    fn borrow(&self) -> &UsernameRef {
+        UsernameRef::new(self.as_str())
+    }
+}
+
+impl AsRef<UsernameRef> for Username {
+    fn as_ref(&self) -> &UsernameRef {
+        UsernameRef::new(self.as_str())
+    }
+}
+
+impl ToOwned for UsernameRef {
+    type Owned = Username;
+
+    fn to_owned(&self) -> Self::Owned {
+        Username(self.0.to_owned())
+    }
+}
+
+impl TryFrom<String> for Username {
+    type Error = Error;
+
+    fn try_from(s: String) -> Result<Self, Error> {
+        if !PROXMOX_USER_NAME_REGEX.is_match(&s) {
+            bail!("invalid user name");
+        }
+
+        Ok(Self(s))
+    }
+}
+
+impl<'a> TryFrom<&'a str> for &'a UsernameRef {
+    type Error = Error;
+
+    fn try_from(s: &'a str) -> Result<&'a UsernameRef, Error> {
+        if !PROXMOX_USER_NAME_REGEX.is_match(s) {
+            bail!("invalid name in user id");
+        }
+
+        Ok(UsernameRef::new(s))
+    }
+}
+
+#[api(schema: PROXMOX_AUTH_REALM_SCHEMA)]
+/// An authentication realm.
+#[derive(Clone, Debug, Eq, PartialEq, Hash, Deserialize, Serialize)]
+pub struct Realm(String);
+
+/// A reference to an authentication realm.
+///
+/// This is like a `str` to the `String` of a `Realm`.
+#[derive(Debug, Hash, Eq, PartialEq)]
+pub struct RealmRef(str);
+
+impl RealmRef {
+    fn new(s: &str) -> &Self {
+        unsafe { &*(s as *const str as *const RealmRef) }
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl std::ops::Deref for Realm {
+    type Target = RealmRef;
+
+    fn deref(&self) -> &RealmRef {
+        self.borrow()
+    }
+}
+
+impl Borrow<RealmRef> for Realm {
+    fn borrow(&self) -> &RealmRef {
+        RealmRef::new(self.as_str())
+    }
+}
+
+impl AsRef<RealmRef> for Realm {
+    fn as_ref(&self) -> &RealmRef {
+        RealmRef::new(self.as_str())
+    }
+}
+
+impl ToOwned for RealmRef {
+    type Owned = Realm;
+
+    fn to_owned(&self) -> Self::Owned {
+        Realm(self.0.to_owned())
+    }
+}
+
+impl TryFrom<String> for Realm {
+    type Error = Error;
+
+    fn try_from(s: String) -> Result<Self, Error> {
+        PROXMOX_AUTH_REALM_STRING_SCHEMA.check_constraints(&s)
+            .map_err(|_| format_err!("invalid realm"))?;
+
+        Ok(Self(s))
+    }
+}
+
+impl<'a> TryFrom<&'a str> for &'a RealmRef {
+    type Error = Error;
+
+    fn try_from(s: &'a str) -> Result<&'a RealmRef, Error> {
+        PROXMOX_AUTH_REALM_STRING_SCHEMA.check_constraints(s)
+            .map_err(|_| format_err!("invalid realm"))?;
+
+        Ok(RealmRef::new(s))
+    }
+}
+
+impl PartialEq<str> for Realm {
+    fn eq(&self, rhs: &str) -> bool {
+        self.0 == rhs
+    }
+}
+
+impl PartialEq<&str> for Realm {
+    fn eq(&self, rhs: &&str) -> bool {
+        self.0 == *rhs
+    }
+}
+
+impl PartialEq<str> for RealmRef {
+    fn eq(&self, rhs: &str) -> bool {
+        self.0 == *rhs
+    }
+}
+
+impl PartialEq<&str> for RealmRef {
+    fn eq(&self, rhs: &&str) -> bool {
+        self.0 == **rhs
+    }
+}
+
+impl PartialEq<RealmRef> for Realm {
+    fn eq(&self, rhs: &RealmRef) -> bool {
+        self.0 == &rhs.0
+    }
+}
+
+impl PartialEq<Realm> for RealmRef {
+    fn eq(&self, rhs: &Realm) -> bool {
+        self.0 == rhs.0
+    }
+}
+
+impl PartialEq<Realm> for &RealmRef {
+    fn eq(&self, rhs: &Realm) -> bool {
+        (*self).0 == rhs.0
+    }
+}
+
+/// A complete user id consting of a user name and a realm.
+#[derive(Clone, Debug, Hash)]
+pub struct Userid {
+    data: String,
+    name_len: usize,
+    //name: Username,
+    //realm: Realm,
+}
+
+impl Userid {
+    pub const API_SCHEMA: Schema = StringSchema::new("User ID")
+        .format(&PROXMOX_USER_ID_FORMAT)
+        .min_length(3)
+        .max_length(64)
+        .schema();
+
+    const fn new(data: String, name_len: usize) -> Self {
+        Self { data, name_len }
+    }
+
+    pub fn name(&self) -> &UsernameRef {
+        UsernameRef::new(&self.data[..self.name_len])
+    }
+
+    pub fn realm(&self) -> &RealmRef {
+        RealmRef::new(&self.data[(self.name_len + 1)..])
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.data
+    }
+
+    /// Get the "backup@pam" user id.
+    pub fn backup_userid() -> &'static Self {
+        &*BACKUP_USERID
+    }
+
+    /// Get the "root@pam" user id.
+    pub fn root_userid() -> &'static Self {
+        &*ROOT_USERID
+    }
+}
+
+lazy_static! {
+    pub static ref BACKUP_USERID: Userid = Userid::new("backup@pam".to_string(), 6);
+    pub static ref ROOT_USERID: Userid = Userid::new("root@pam".to_string(), 4);
+}
+
+impl Eq for Userid {}
+
+impl PartialEq for Userid {
+    fn eq(&self, rhs: &Self) -> bool {
+        self.data == rhs.data && self.name_len == rhs.name_len
+    }
+}
+
+impl From<(Username, Realm)> for Userid {
+    fn from(parts: (Username, Realm)) -> Self {
+        Self::from((parts.0.as_ref(), parts.1.as_ref()))
+    }
+}
+
+impl From<(&UsernameRef, &RealmRef)> for Userid {
+    fn from(parts: (&UsernameRef, &RealmRef)) -> Self {
+        let data = format!("{}@{}", parts.0.as_str(), parts.1.as_str());
+        let name_len = parts.0.as_str().len();
+        Self { data, name_len }
+    }
+}
+
+impl fmt::Display for Userid {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.data.fmt(f)
+    }
+}
+
+impl std::str::FromStr for Userid {
+    type Err = Error;
+
+    fn from_str(id: &str) -> Result<Self, Error> {
+        let (name, realm) = match id.as_bytes().iter().rposition(|&b| b == b'@') {
+            Some(pos) => (&id[..pos], &id[(pos + 1)..]),
+            None => bail!("not a valid user id"),
+        };
+
+        PROXMOX_AUTH_REALM_STRING_SCHEMA.check_constraints(realm)
+            .map_err(|_| format_err!("invalid realm in user id"))?;
+
+        Ok(Self::from((UsernameRef::new(name), RealmRef::new(realm))))
+    }
+}
+
+impl TryFrom<String> for Userid {
+    type Error = Error;
+
+    fn try_from(data: String) -> Result<Self, Error> {
+        let name_len = data
+            .as_bytes()
+            .iter()
+            .rposition(|&b| b == b'@')
+            .ok_or_else(|| format_err!("not a valid user id"))?;
+
+        PROXMOX_AUTH_REALM_STRING_SCHEMA.check_constraints(&data[(name_len + 1)..])
+            .map_err(|_| format_err!("invalid realm in user id"))?;
+
+        Ok(Self { data, name_len })
+    }
+}
+
+impl PartialEq<str> for Userid {
+    fn eq(&self, rhs: &str) -> bool {
+        rhs.len() > self.name_len + 2 // make sure range access below is allowed
+        && rhs.starts_with(self.name().as_str())
+        && rhs.as_bytes()[self.name_len] == b'@'
+        && &rhs[(self.name_len + 1)..] == self.realm().as_str()
+    }
+}
+
+impl PartialEq<&str> for Userid {
+    fn eq(&self, rhs: &&str) -> bool {
+        *self == **rhs
+    }
+}
+
+impl PartialEq<String> for Userid {
+    fn eq(&self, rhs: &String) -> bool {
+        self == rhs.as_str()
+    }
+}
+
+proxmox::forward_deserialize_to_from_str!(Userid);
+proxmox::forward_serialize_to_display!(Userid);

src/auth.rs

@@ -10,39 +10,54 @@ use base64;
 use anyhow::{bail, format_err, Error};
 use serde_json::json;
 
+use crate::api2::types::{Userid, UsernameRef, RealmRef};
+
 pub trait ProxmoxAuthenticator {
-    fn authenticate_user(&self, username: &str, password: &str) -> Result<(), Error>;
-    fn store_password(&self, username: &str, password: &str) -> Result<(), Error>;
+    fn authenticate_user(&self, username: &UsernameRef, password: &str) -> Result<(), Error>;
+    fn store_password(&self, username: &UsernameRef, password: &str) -> Result<(), Error>;
 }
 
 pub struct PAM();
 
 impl ProxmoxAuthenticator for PAM {
 
-    fn authenticate_user(&self, username: &str, password: &str) -> Result<(), Error> {
+    fn authenticate_user(&self, username: &UsernameRef, password: &str) -> Result<(), Error> {
         let mut auth = pam::Authenticator::with_password("proxmox-backup-auth").unwrap();
-        auth.get_handler().set_credentials(username, password);
+        auth.get_handler().set_credentials(username.as_str(), password);
         auth.authenticate()?;
         return Ok(());
 
     }
 
-    fn store_password(&self, username: &str, password: &str) -> Result<(), Error> {
+    fn store_password(&self, username: &UsernameRef, password: &str) -> Result<(), Error> {
         let mut child = Command::new("passwd")
-            .arg(username)
+            .arg(username.as_str())
             .stdin(Stdio::piped())
             .stderr(Stdio::piped())
             .spawn()
-            .or_else(|err| Err(format_err!("unable to set password for '{}' - execute passwd failed: {}", username, err)))?;
+            .map_err(|err| format_err!(
+                "unable to set password for '{}' - execute passwd failed: {}",
+                username.as_str(),
+                err,
+            ))?;
 
         // Note: passwd reads password twice from stdin (for verify)
         writeln!(child.stdin.as_mut().unwrap(), "{}\n{}", password, password)?;
 
-        let output = child.wait_with_output()
-            .or_else(|err| Err(format_err!("unable to set password for '{}' - wait failed: {}", username, err)))?;
+        let output = child
+            .wait_with_output()
+            .map_err(|err| format_err!(
+                "unable to set password for '{}' - wait failed: {}",
+                username.as_str(),
+                err,
+            ))?;
 
         if !output.status.success() {
-            bail!("unable to set password for '{}' - {}", username, String::from_utf8_lossy(&output.stderr));
+            bail!(
+                "unable to set password for '{}' - {}",
+                username.as_str(),
+                String::from_utf8_lossy(&output.stderr),
+            );
         }
 
         Ok(())
@@ -90,23 +105,23 @@ pub fn verify_crypt_pw(password: &str, enc_password: &str) -> Result<(), Error>
     Ok(())
 }
 
-const SHADOW_CONFIG_FILENAME: &str = "/etc/proxmox-backup/shadow.json";
+const SHADOW_CONFIG_FILENAME: &str = configdir!("/shadow.json");
 
 impl ProxmoxAuthenticator for PBS {
 
-    fn authenticate_user(&self, username: &str, password: &str) -> Result<(), Error> {
+    fn authenticate_user(&self, username: &UsernameRef, password: &str) -> Result<(), Error> {
         let data = proxmox::tools::fs::file_get_json(SHADOW_CONFIG_FILENAME, Some(json!({})))?;
-        match data[username].as_str() {
+        match data[username.as_str()].as_str() {
             None => bail!("no password set"),
             Some(enc_password) => verify_crypt_pw(password, enc_password)?,
         }
         Ok(())
     }
 
-    fn store_password(&self, username: &str, password: &str) -> Result<(), Error> {
+    fn store_password(&self, username: &UsernameRef, password: &str) -> Result<(), Error> {
         let enc_password = encrypt_pw(password)?;
         let mut data = proxmox::tools::fs::file_get_json(SHADOW_CONFIG_FILENAME, Some(json!({})))?;
-        data[username] = enc_password.into();
+        data[username.as_str()] = enc_password.into();
 
         let mode = nix::sys::stat::Mode::from_bits_truncate(0o0600);
         let options =  proxmox::tools::fs::CreateOptions::new()
@@ -121,28 +136,18 @@ impl ProxmoxAuthenticator for PBS {
     }
 }
 
-pub fn parse_userid(userid: &str) -> Result<(String, String), Error> {
-    let data: Vec<&str> = userid.rsplitn(2, '@').collect();
-
-    if data.len() != 2 {
-        bail!("userid '{}' has no realm", userid);
-    }
-    Ok((data[1].to_owned(), data[0].to_owned()))
-}
-
 /// Lookup the autenticator for the specified realm
-pub fn lookup_authenticator(realm: &str) -> Result<Box<dyn ProxmoxAuthenticator>, Error> {
-    match realm {
+pub fn lookup_authenticator(realm: &RealmRef) -> Result<Box<dyn ProxmoxAuthenticator>, Error> {
+    match realm.as_str() {
         "pam" => Ok(Box::new(PAM())),
         "pbs" => Ok(Box::new(PBS())),
-        _ => bail!("unknown realm '{}'", realm),
+        _ => bail!("unknown realm '{}'", realm.as_str()),
     }
 }
 
 /// Authenticate users
-pub fn authenticate_user(userid: &str, password: &str) -> Result<(), Error> {
-    let (username, realm) = parse_userid(userid)?;
+pub fn authenticate_user(userid: &Userid, password: &str) -> Result<(), Error> {
 
-    lookup_authenticator(&realm)?
-    .authenticate_user(&username, password)
+    lookup_authenticator(userid.realm())?
+        .authenticate_user(userid.name(), password)
 }

src/auth_helpers.rs

@@ -10,14 +10,17 @@ use std::path::PathBuf;
 use proxmox::tools::fs::{file_get_contents, replace_file, CreateOptions};
 use proxmox::try_block;
 
+use crate::api2::types::Userid;
+use crate::tools::epoch_now_u64;
+
 fn compute_csrf_secret_digest(
     timestamp: i64,
     secret: &[u8],
-    username: &str,
+    userid: &Userid,
 ) -> String {
 
     let mut hasher = sha::Sha256::new();
-    let data = format!("{:08X}:{}:", timestamp, username);
+    let data = format!("{:08X}:{}:", timestamp, userid);
     hasher.update(data.as_bytes());
     hasher.update(secret);
 
@@ -26,20 +29,19 @@ fn compute_csrf_secret_digest(
 
 pub fn assemble_csrf_prevention_token(
     secret: &[u8],
-    username: &str,
+    userid: &Userid,
 ) -> String {
 
-    let epoch = std::time::SystemTime::now().duration_since(
-        std::time::SystemTime::UNIX_EPOCH).unwrap().as_secs() as i64;
+    let epoch = epoch_now_u64().unwrap() as i64;
 
-    let digest = compute_csrf_secret_digest(epoch, secret, username);
+    let digest = compute_csrf_secret_digest(epoch, secret, userid);
 
     format!("{:08X}:{}", epoch, digest)
 }
 
 pub fn verify_csrf_prevention_token(
     secret: &[u8],
-    username: &str,
+    userid: &Userid,
     token: &str,
     min_age: i64,
     max_age: i64,
@@ -61,14 +63,13 @@ pub fn verify_csrf_prevention_token(
         let ttime = i64::from_str_radix(timestamp, 16).
             map_err(|err| format_err!("timestamp format error - {}", err))?;
 
-        let digest = compute_csrf_secret_digest(ttime, secret, username);
+        let digest = compute_csrf_secret_digest(ttime, secret, userid);
 
         if digest != sig {
             bail!("invalid signature.");
         }
 
-        let now = std::time::SystemTime::now().duration_since(
-            std::time::SystemTime::UNIX_EPOCH)?.as_secs() as i64;
+        let now = epoch_now_u64()? as i64;
 
         let age = now - ttime;
         if age < min_age {

src/backup.rs

@@ -40,21 +40,21 @@
 //!
 //!   Acquire shared lock for ChunkStore (process wide).
 //!
-//!   Note: When creating .idx files, we create temporary (.tmp) file,
+//!   Note: When creating .idx files, we create temporary a (.tmp) file,
 //!   then do an atomic rename ...
 //!
 //!
 //! * Garbage Collect:
 //!
 //!   Acquire exclusive lock for ChunkStore (process wide). If we have
-//!   already an shared lock for ChunkStore, try to updraged that
+//!   already a shared lock for the ChunkStore, try to upgrade that
 //!   lock.
 //!
 //!
 //! * Server Restart
 //!
-//!   Try to abort running garbage collection to release exclusive
-//!   ChunkStore lock asap. Start new service with existing listening
+//!   Try to abort the running garbage collection to release exclusive
+//!   ChunkStore locks ASAP. Start the new service with the existing listening
 //!   socket.
 //!
 //!
@@ -62,10 +62,10 @@
 //!
 //! Deleting backups is as easy as deleting the corresponding .idx
 //! files. Unfortunately, this does not free up any storage, because
-//! those files just contains references to chunks.
+//! those files just contain references to chunks.
 //!
 //! To free up some storage, we run a garbage collection process at
-//! regular intervals. The collector uses an mark and sweep
+//! regular intervals. The collector uses a mark and sweep
 //! approach. In the first phase, it scans all .idx files to mark used
 //! chunks. The second phase then removes all unmarked chunks from the
 //! store.
@@ -90,12 +90,12 @@
 //! amount of time ago (by default 24h). So we may only delete chunks
 //! with `atime` older than 24 hours.
 //!
-//! Another problem arise from running backups. The mark phase does
+//! Another problem arises from running backups. The mark phase does
 //! not find any chunks from those backups, because there is no .idx
 //! file for them (created after the backup). Chunks created or
 //! touched by those backups may have an `atime` as old as the start
-//! time of those backup. Please not that the backup start time may
-//! predate the GC start time. Se we may only delete chunk older than
+//! time of those backups. Please note that the backup start time may
+//! predate the GC start time. So we may only delete chunks older than
 //! the start time of those running backup jobs.
 //!
 //!
@@ -120,6 +120,8 @@ macro_rules! PROXMOX_BACKUP_READER_PROTOCOL_ID_V1 {
 
 /// Unix system user used by proxmox-backup-proxy
 pub const BACKUP_USER_NAME: &str = "backup";
+/// Unix system group used by proxmox-backup-proxy
+pub const BACKUP_GROUP_NAME: &str = "backup";
 
 /// Return User info for the 'backup' user (``getpwnam_r(3)``)
 pub fn backup_user() -> Result<nix::unistd::User, Error> {
@@ -129,6 +131,14 @@ pub fn backup_user() -> Result<nix::unistd::User, Error> {
     }
 }
 
+/// Return Group info for the 'backup' group (``getgrnam(3)``)
+pub fn backup_group() -> Result<nix::unistd::Group, Error> {
+    match nix::unistd::Group::from_name(BACKUP_GROUP_NAME)? {
+        Some(group) => Ok(group),
+        None => bail!("Unable to lookup backup user."),
+    }
+}
+
 mod file_formats;
 pub use file_formats::*;
 
@@ -198,5 +208,11 @@ pub use prune::*;
 mod datastore;
 pub use datastore::*;
 
+mod verify;
+pub use verify::*;
+
 mod catalog_shell;
 pub use catalog_shell::*;
+
+mod async_index_reader;
+pub use async_index_reader::*;

src/backup/async_index_reader.rs

@@ -0,0 +1,204 @@
+use std::future::Future;
+use std::task::{Poll, Context};
+use std::pin::Pin;
+use std::io::SeekFrom;
+
+use anyhow::Error;
+use futures::future::FutureExt;
+use futures::ready;
+use tokio::io::{AsyncRead, AsyncSeek};
+
+use proxmox::sys::error::io_err_other;
+use proxmox::io_format_err;
+
+use super::IndexFile;
+use super::read_chunk::AsyncReadChunk;
+use super::index::ChunkReadInfo;
+
+enum AsyncIndexReaderState<S> {
+    NoData,
+    WaitForData(Pin<Box<dyn Future<Output = Result<(S, Vec<u8>), Error>> + Send + 'static>>),
+    HaveData,
+}
+
+pub struct AsyncIndexReader<S, I: IndexFile> {
+    store: Option<S>,
+    index: I,
+    read_buffer: Vec<u8>,
+    current_chunk_offset: u64,
+    current_chunk_idx: usize,
+    current_chunk_info: Option<ChunkReadInfo>,
+    position: u64,
+    seek_to_pos: i64,
+    state: AsyncIndexReaderState<S>,
+}
+
+// ok because the only public interfaces operates on &mut Self
+unsafe impl<S: Sync, I: IndexFile + Sync> Sync for AsyncIndexReader<S, I> {}
+
+impl<S: AsyncReadChunk, I: IndexFile> AsyncIndexReader<S, I> {
+    pub fn new(index: I, store: S) -> Self {
+        Self {
+            store: Some(store),
+            index,
+            read_buffer: Vec::with_capacity(1024 * 1024),
+            current_chunk_offset: 0,
+            current_chunk_idx: 0,
+            current_chunk_info: None,
+            position: 0,
+            seek_to_pos: 0,
+            state: AsyncIndexReaderState::NoData,
+        }
+    }
+}
+
+impl<S, I> AsyncRead for AsyncIndexReader<S, I>
+where
+    S: AsyncReadChunk + Unpin + Sync + 'static,
+    I: IndexFile + Unpin,
+{
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context,
+        buf: &mut [u8],
+    ) -> Poll<tokio::io::Result<usize>> {
+        let this = Pin::get_mut(self);
+        loop {
+            match &mut this.state {
+                AsyncIndexReaderState::NoData => {
+                    let (idx, offset) = if this.current_chunk_info.is_some() &&
+                        this.position == this.current_chunk_info.as_ref().unwrap().range.end
+                    {
+                        // optimization for sequential chunk read
+                        let next_idx = this.current_chunk_idx + 1;
+                        (next_idx, 0)
+                    } else {
+                        match this.index.chunk_from_offset(this.position) {
+                            Some(res) => res,
+                            None => return Poll::Ready(Ok(0))
+                        }
+                    };
+
+                    if idx >= this.index.index_count() {
+                        return Poll::Ready(Ok(0));
+                    }
+
+                    let info = this
+                        .index
+                        .chunk_info(idx)
+                        .ok_or(io_format_err!("could not get digest"))?;
+
+                    this.current_chunk_offset = offset;
+                    this.current_chunk_idx = idx;
+                    let old_info = this.current_chunk_info.replace(info.clone());
+
+                    if let Some(old_info) = old_info {
+                        if old_info.digest == info.digest {
+                            // hit, chunk is currently in cache
+                            this.state = AsyncIndexReaderState::HaveData;
+                            continue;
+                        }
+                    }
+
+                    // miss, need to download new chunk
+                    let store = match this.store.take() {
+                        Some(store) => store,
+                        None => {
+                            return Poll::Ready(Err(io_format_err!("could not find store")));
+                        }
+                    };
+
+                    let future = async move {
+                        store.read_chunk(&info.digest)
+                            .await
+                            .map(move |x| (store, x))
+                    };
+
+                    this.state = AsyncIndexReaderState::WaitForData(future.boxed());
+                }
+                AsyncIndexReaderState::WaitForData(ref mut future) => {
+                    match ready!(future.as_mut().poll(cx)) {
+                        Ok((store, mut chunk_data)) => {
+                            this.read_buffer.clear();
+                            this.read_buffer.append(&mut chunk_data);
+                            this.state = AsyncIndexReaderState::HaveData;
+                            this.store = Some(store);
+                        }
+                        Err(err) => {
+                            return Poll::Ready(Err(io_err_other(err)));
+                        }
+                    };
+                }
+                AsyncIndexReaderState::HaveData => {
+                    let offset = this.current_chunk_offset as usize;
+                    let len = this.read_buffer.len();
+                    let n = if len - offset < buf.len() {
+                        len - offset
+                    } else {
+                        buf.len()
+                    };
+
+                    buf[0..n].copy_from_slice(&this.read_buffer[offset..(offset + n)]);
+                    this.position += n as u64;
+
+                    if offset + n == len {
+                        this.state = AsyncIndexReaderState::NoData;
+                    } else {
+                        this.current_chunk_offset += n as u64;
+                        this.state = AsyncIndexReaderState::HaveData;
+                    }
+
+                    return Poll::Ready(Ok(n));
+                }
+            }
+        }
+    }
+}
+
+impl<S, I> AsyncSeek for AsyncIndexReader<S, I>
+where
+    S: AsyncReadChunk + Unpin + Sync + 'static,
+    I: IndexFile + Unpin,
+{
+    fn start_seek(
+        self: Pin<&mut Self>,
+        _cx: &mut Context<'_>,
+        pos: SeekFrom,
+    ) -> Poll<tokio::io::Result<()>> {
+        let this = Pin::get_mut(self);
+        this.seek_to_pos = match pos {
+            SeekFrom::Start(offset) => {
+                offset as i64
+            },
+            SeekFrom::End(offset) => {
+                this.index.index_bytes() as i64 + offset
+            },
+            SeekFrom::Current(offset) => {
+                this.position as i64 + offset
+            }
+        };
+        Poll::Ready(Ok(()))
+    }
+
+    fn poll_complete(
+        self: Pin<&mut Self>,
+        _cx: &mut Context<'_>,
+    ) -> Poll<tokio::io::Result<u64>> {
+        let this = Pin::get_mut(self);
+
+        let index_bytes = this.index.index_bytes();
+        if this.seek_to_pos < 0 {
+            return Poll::Ready(Err(io_format_err!("cannot seek to negative values")));
+        } else if this.seek_to_pos > index_bytes as i64 {
+            this.position = index_bytes;
+        } else {
+            this.position = this.seek_to_pos as u64;
+        }
+
+        // even if seeking within one chunk, we need to go to NoData to
+        // recalculate the current_chunk_offset (data is cached anyway)
+        this.state = AsyncIndexReaderState::NoData;
+
+        Poll::Ready(Ok(this.position))
+    }
+}

src/backup/backup_info.rs

@@ -45,6 +45,31 @@ pub struct BackupGroup {
     backup_id: String,
 }
 
+impl std::cmp::Ord for BackupGroup {
+
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        let type_order = self.backup_type.cmp(&other.backup_type);
+        if type_order != std::cmp::Ordering::Equal {
+            return type_order;
+        }
+        // try to compare IDs numerically
+        let id_self = self.backup_id.parse::<u64>();
+        let id_other = other.backup_id.parse::<u64>();
+        match (id_self, id_other) {
+            (Ok(id_self), Ok(id_other)) => id_self.cmp(&id_other),
+            (Ok(_), Err(_)) => std::cmp::Ordering::Less,
+            (Err(_), Ok(_)) => std::cmp::Ordering::Greater,
+             _ => self.backup_id.cmp(&other.backup_id),
+        }
+    }
+}
+
+impl std::cmp::PartialOrd for BackupGroup {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 impl BackupGroup {
 
     pub fn new<T: Into<String>, U: Into<String>>(backup_type: T, backup_id: U) -> Self {
@@ -59,17 +84,6 @@ impl BackupGroup {
         &self.backup_id
     }
 
-    pub fn parse(path: &str) -> Result<Self, Error> {
-
-        let cap = GROUP_PATH_REGEX.captures(path)
-            .ok_or_else(|| format_err!("unable to parse backup group path '{}'", path))?;
-
-        Ok(Self {
-            backup_type: cap.get(1).unwrap().as_str().to_owned(),
-            backup_id: cap.get(2).unwrap().as_str().to_owned(),
-        })
-    }
-
     pub fn group_path(&self) ->  PathBuf  {
 
         let mut relative_path = PathBuf::new();
@@ -117,7 +131,11 @@ impl BackupGroup {
 
             use nix::fcntl::{openat, OFlag};
             match openat(l2_fd, &manifest_path, OFlag::O_RDONLY, nix::sys::stat::Mode::empty()) {
-                Ok(_) => { /* manifest exists --> assume backup was successful */ },
+                Ok(rawfd) => {
+                    /* manifest exists --> assume backup was successful */
+                    /* close else this leaks! */
+                    nix::unistd::close(rawfd)?;
+                },
                 Err(nix::Error::Sys(nix::errno::Errno::ENOENT)) => { return Ok(()); }
                 Err(err) => {
                     bail!("last_successful_backup: unexpected error - {}", err);
@@ -152,10 +170,35 @@ impl BackupGroup {
     }
 }
 
+impl std::fmt::Display for BackupGroup {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let backup_type = self.backup_type();
+        let id = self.backup_id();
+        write!(f, "{}/{}", backup_type, id)
+    }
+}
+
+impl std::str::FromStr for BackupGroup {
+    type Err = Error;
+
+    /// Parse a backup group path
+    ///
+    /// This parses strings like `vm/100".
+    fn from_str(path: &str) -> Result<Self, Self::Err> {
+        let cap = GROUP_PATH_REGEX.captures(path)
+            .ok_or_else(|| format_err!("unable to parse backup group path '{}'", path))?;
+
+        Ok(Self {
+            backup_type: cap.get(1).unwrap().as_str().to_owned(),
+            backup_id: cap.get(2).unwrap().as_str().to_owned(),
+        })
+    }
+}
+
 /// Uniquely identify a Backup (relative to data store)
 ///
 /// We also call this a backup snaphost.
-#[derive(Debug, Clone)]
+#[derive(Debug, Eq, PartialEq, Clone)]
 pub struct BackupDir {
     /// Backup group
     group: BackupGroup,
@@ -188,16 +231,6 @@ impl BackupDir {
         self.backup_time
     }
 
-    pub fn parse(path: &str) -> Result<Self, Error> {
-
-        let cap = SNAPSHOT_PATH_REGEX.captures(path)
-            .ok_or_else(|| format_err!("unable to parse backup snapshot path '{}'", path))?;
-
-        let group = BackupGroup::new(cap.get(1).unwrap().as_str(), cap.get(2).unwrap().as_str());
-        let backup_time = cap.get(3).unwrap().as_str().parse::<DateTime<Utc>>()?;
-        Ok(BackupDir::from((group, backup_time.timestamp())))
-    }
-
     pub fn relative_path(&self) ->  PathBuf  {
 
         let mut relative_path = self.group.group_path();
@@ -212,6 +245,31 @@ impl BackupDir {
     }
 }
 
+impl std::str::FromStr for BackupDir {
+    type Err = Error;
+
+    /// Parse a snapshot path
+    ///
+    /// This parses strings like `host/elsa/2020-06-15T05:18:33Z".
+    fn from_str(path: &str) -> Result<Self, Self::Err> {
+        let cap = SNAPSHOT_PATH_REGEX.captures(path)
+            .ok_or_else(|| format_err!("unable to parse backup snapshot path '{}'", path))?;
+
+        let group = BackupGroup::new(cap.get(1).unwrap().as_str(), cap.get(2).unwrap().as_str());
+        let backup_time = cap.get(3).unwrap().as_str().parse::<DateTime<Utc>>()?;
+        Ok(BackupDir::from((group, backup_time.timestamp())))
+    }
+}
+
+impl std::fmt::Display for BackupDir {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let backup_type = self.group.backup_type();
+        let id = self.group.backup_id();
+        let time = Self::backup_time_to_string(self.backup_time);
+        write!(f, "{}/{}/{}", backup_type, id, time)
+    }
+}
+
 impl From<(BackupGroup, i64)> for BackupDir {
     fn from((group, timestamp): (BackupGroup, i64)) -> Self {
         Self { group, backup_time: Utc.timestamp(timestamp, 0) }
@@ -239,9 +297,13 @@ impl BackupInfo {
     }
 
     /// Finds the latest backup inside a backup group
-    pub fn last_backup(base_path: &Path, group: &BackupGroup) -> Result<Option<BackupInfo>, Error> {
+    pub fn last_backup(base_path: &Path, group: &BackupGroup, only_finished: bool)
+        -> Result<Option<BackupInfo>, Error>
+    {
         let backups = group.list_backups(base_path)?;
-        Ok(backups.into_iter().max_by_key(|item| item.backup_dir.backup_time()))
+        Ok(backups.into_iter()
+            .filter(|item| !only_finished || item.is_finished())
+            .max_by_key(|item| item.backup_dir.backup_time()))
     }
 
     pub fn sort_list(list: &mut Vec<BackupInfo>, ascendending: bool) {
@@ -284,6 +346,11 @@ impl BackupInfo {
         })?;
         Ok(list)
     }
+
+    pub fn is_finished(&self) -> bool {
+        // backup is considered unfinished if there is no manifest
+        self.files.iter().any(|name| name == super::MANIFEST_BLOB_NAME)
+    }
 }
 
 fn list_backup_files<P: ?Sized + nix::NixPath>(dirfd: RawFd, path: &P) -> Result<Vec<String>, Error> {

src/backup/catalog.rs

@@ -1,23 +1,21 @@
-use anyhow::{bail, format_err, Error};
-use std::fmt;
-use std::ffi::{CStr, CString, OsStr};
-use std::os::unix::ffi::OsStrExt;
-use std::io::{Read, Write, Seek, SeekFrom};
 use std::convert::TryFrom;
+use std::ffi::{CStr, CString, OsStr};
+use std::fmt;
+use std::io::{Read, Write, Seek, SeekFrom};
+use std::os::unix::ffi::OsStrExt;
 
+use anyhow::{bail, format_err, Error};
 use chrono::offset::{TimeZone, Local};
 
+use pathpatterns::{MatchList, MatchType};
 use proxmox::tools::io::ReadExt;
-use proxmox::sys::error::io_err_other;
 
-use crate::pxar::catalog::BackupCatalogWriter;
-use crate::pxar::{MatchPattern, MatchPatternSlice, MatchType};
 use crate::backup::file_formats::PROXMOX_CATALOG_FILE_MAGIC_1_0;
-use crate::tools::runtime::block_on;
+use crate::pxar::catalog::BackupCatalogWriter;
 
 #[repr(u8)]
 #[derive(Copy,Clone,PartialEq)]
-enum CatalogEntryType {
+pub(crate) enum CatalogEntryType {
     Directory = b'd',
     File = b'f',
     Symlink = b'l',
@@ -46,6 +44,21 @@ impl TryFrom<u8> for CatalogEntryType {
     }
 }
 
+impl From<&DirEntryAttribute> for CatalogEntryType {
+    fn from(value: &DirEntryAttribute) -> Self {
+        match value {
+            DirEntryAttribute::Directory { .. } => CatalogEntryType::Directory,
+            DirEntryAttribute::File { .. } => CatalogEntryType::File,
+            DirEntryAttribute::Symlink => CatalogEntryType::Symlink,
+            DirEntryAttribute::Hardlink => CatalogEntryType::Hardlink,
+            DirEntryAttribute::BlockDevice => CatalogEntryType::BlockDevice,
+            DirEntryAttribute::CharDevice => CatalogEntryType::CharDevice,
+            DirEntryAttribute::Fifo => CatalogEntryType::Fifo,
+            DirEntryAttribute::Socket => CatalogEntryType::Socket,
+        }
+    }
+}
+
 impl fmt::Display for CatalogEntryType {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{}", char::from(*self as u8))
@@ -63,7 +76,7 @@ pub struct DirEntry {
 }
 
 /// Used to specific additional attributes inside DirEntry
-#[derive(Clone, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum DirEntryAttribute {
     Directory { start: u64 },
     File { size: u64, mtime: u64 },
@@ -106,6 +119,23 @@ impl DirEntry {
         }
     }
 
+    /// Get file mode bits for this entry to be used with the `MatchList` api.
+    pub fn get_file_mode(&self) -> Option<u32> {
+        Some(
+            match self.attr {
+                DirEntryAttribute::Directory { .. } => pxar::mode::IFDIR,
+                DirEntryAttribute::File { .. } => pxar::mode::IFREG,
+                DirEntryAttribute::Symlink => pxar::mode::IFLNK,
+                DirEntryAttribute::Hardlink => return None,
+                DirEntryAttribute::BlockDevice => pxar::mode::IFBLK,
+                DirEntryAttribute::CharDevice => pxar::mode::IFCHR,
+                DirEntryAttribute::Fifo => pxar::mode::IFIFO,
+                DirEntryAttribute::Socket => pxar::mode::IFSOCK,
+            }
+            as u32
+        )
+    }
+
     /// Check if DirEntry is a directory
     pub fn is_directory(&self) -> bool {
         match self.attr {
@@ -383,32 +413,6 @@ impl <W: Write> BackupCatalogWriter for CatalogWriter<W> {
     }
 }
 
-// fixme: move to somehere else?
-/// Implement Write to tokio mpsc channel Sender
-pub struct SenderWriter(tokio::sync::mpsc::Sender<Result<Vec<u8>, Error>>);
-
-impl SenderWriter {
-    pub fn new(sender: tokio::sync::mpsc::Sender<Result<Vec<u8>, Error>>) -> Self {
-        Self(sender)
-    }
-}
-
-impl Write for SenderWriter {
-    fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
-        block_on(async move {
-            self.0
-                .send(Ok(buf.to_vec()))
-                .await
-                .map_err(io_err_other)
-                .and(Ok(buf.len()))
-        })
-    }
-
-    fn flush(&mut self) -> Result<(), std::io::Error> {
-        Ok(())
-    }
-}
-
 /// Read Catalog files
 pub struct CatalogReader<R> {
     reader: R,
@@ -476,7 +480,7 @@ impl <R: Read + Seek> CatalogReader<R> {
         &mut self,
         parent: &DirEntry,
         filename: &[u8],
-    ) -> Result<DirEntry, Error>  {
+    ) -> Result<Option<DirEntry>, Error>  {
 
         let start = match parent.attr {
             DirEntryAttribute::Directory { start } => start,
@@ -496,10 +500,7 @@ impl <R: Read + Seek> CatalogReader<R> {
             Ok(false) // stop parsing
         })?;
 
-        match item {
-            None => bail!("no such file"),
-            Some(entry) => Ok(entry),
-        }
+        Ok(item)
     }
 
     /// Read the raw directory info block from current reader position.
@@ -532,7 +533,10 @@ impl <R: Read + Seek> CatalogReader<R> {
                     self.dump_dir(&path, pos)?;
                 }
                 CatalogEntryType::File => {
-                    let dt = Local.timestamp(mtime as i64, 0);
+                    let dt = Local
+                        .timestamp_opt(mtime as i64, 0)
+                        .single() // chrono docs say timestamp_opt can only be None or Single!
+                        .unwrap_or_else(|| Local.timestamp(0, 0));
 
                     println!(
                         "{} {:?} {} {}",
@@ -555,38 +559,30 @@ impl <R: Read + Seek> CatalogReader<R> {
     /// provided callback on them.
     pub fn find(
         &mut self,
-        mut entry: &mut Vec<DirEntry>,
-        pattern: &[MatchPatternSlice],
-        callback: &Box<fn(&[DirEntry])>,
+        parent: &DirEntry,
+        file_path: &mut Vec<u8>,
+        match_list: &impl MatchList, //&[MatchEntry],
+        callback: &mut dyn FnMut(&[u8]) -> Result<(), Error>,
     ) -> Result<(), Error> {
-        let parent = entry.last().unwrap();
-        if !parent.is_directory() {
-            return Ok(())
-        }
-
+        let file_len = file_path.len();
         for e in self.read_dir(parent)?  {
-            match MatchPatternSlice::match_filename_include(
-                &CString::new(e.name.clone())?,
-                e.is_directory(),
-                pattern,
-            )? {
-                (MatchType::Positive, _) => {
-                    entry.push(e);
-                    callback(&entry);
-                    let pattern = MatchPattern::from_line(b"**/*").unwrap().unwrap();
-                    let child_pattern = vec![pattern.as_slice()];
-                    self.find(&mut entry, &child_pattern, callback)?;
-                    entry.pop();
-                }
-                (MatchType::PartialPositive, child_pattern)
-                | (MatchType::PartialNegative, child_pattern) => {
-                    entry.push(e);
-                    self.find(&mut entry, &child_pattern, callback)?;
-                    entry.pop();
-                }
-                _ => {}
+            let is_dir = e.is_directory();
+            file_path.truncate(file_len);
+            if !e.name.starts_with(b"/") {
+                file_path.reserve(e.name.len() + 1);
+                file_path.push(b'/');
+            }
+            file_path.extend(&e.name);
+            match match_list.matches(&file_path, e.get_file_mode()) {
+                Some(MatchType::Exclude) => continue,
+                Some(MatchType::Include) => callback(&file_path)?,
+                None => (),
+            }
+            if is_dir {
+                self.find(&e, file_path, match_list, callback)?;
             }
         }
+        file_path.truncate(file_len);
 
         Ok(())
     }

src/backup/chunk_store.rs

@@ -80,8 +80,9 @@ impl ChunkStore {
 
         let default_options = CreateOptions::new();
 
-        if let Err(err) = create_path(&base, Some(default_options.clone()), Some(options.clone())) {
-            bail!("unable to create chunk store '{}' at {:?} - {}", name, base, err);
+        match create_path(&base, Some(default_options.clone()), Some(options.clone())) {
+            Err(err) => bail!("unable to create chunk store '{}' at {:?} - {}", name, base, err),
+            Ok(res) => if ! res  { nix::unistd::chown(&base, Some(uid), Some(gid))? },
         }
 
         if let Err(err) = create_dir(&chunk_dir, options.clone()) {
@@ -103,7 +104,7 @@ impl ChunkStore {
             }
             let percentage = (i*100)/(64*1024);
             if percentage != last_percentage {
-                eprintln!("Percentage done: {}", percentage);
+                eprintln!("{}%", percentage);
                 last_percentage = percentage;
             }
         }
@@ -177,28 +178,12 @@ impl ChunkStore {
                 return Ok(false);
             }
 
-            bail!("updata atime failed for chunk {:?} - {}", chunk_path, err);
+            bail!("update atime failed for chunk {:?} - {}", chunk_path, err);
         }
 
         Ok(true)
     }
 
-    pub fn read_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> {
-
-        let (chunk_path, digest_str) = self.chunk_path(digest);
-        let mut file = std::fs::File::open(&chunk_path)
-            .map_err(|err| {
-                format_err!(
-                    "store '{}', unable to read chunk '{}' - {}",
-                    self.name,
-                    digest_str,
-                    err,
-                )
-            })?;
-
-        DataBlob::load(&mut file)
-    }
-
     pub fn get_chunk_iterator(
         &self,
     ) -> Result<
@@ -290,14 +275,13 @@ impl ChunkStore {
     pub fn sweep_unused_chunks(
         &self,
         oldest_writer: i64,
+        phase1_start_time: i64,
         status: &mut GarbageCollectionStatus,
         worker: &WorkerTask,
     ) -> Result<(), Error> {
         use nix::sys::stat::fstatat;
 
-        let now = unsafe { libc::time(std::ptr::null_mut()) };
-
-        let mut min_atime = now - 3600*24; // at least 24h (see mount option relatime)
+        let mut min_atime = phase1_start_time - 3600*24; // at least 24h (see mount option relatime)
 
         if oldest_writer < min_atime {
             min_atime = oldest_writer;
@@ -311,7 +295,7 @@ impl ChunkStore {
         for (entry, percentage) in self.get_chunk_iterator()? {
             if last_percentage != percentage {
                 last_percentage = percentage;
-                worker.log(format!("percentage done: {}, chunk count: {}", percentage, chunk_count));
+                worker.log(format!("{}%, processed {} chunks", percentage, chunk_count));
             }
 
             worker.fail_on_abort()?;
@@ -429,6 +413,10 @@ impl ChunkStore {
         full_path
     }
 
+    pub fn name(&self) -> &str {
+        &self.name
+    }
+
     pub fn base_path(&self) -> PathBuf {
         self.base.clone()
     }

src/backup/chunker.rs

@@ -5,15 +5,15 @@
 /// use hash value 0 to detect a boundary.
 const CA_CHUNKER_WINDOW_SIZE: usize = 64;
 
-/// Slinding window chunker (Buzhash)
+/// Sliding window chunker (Buzhash)
 ///
 /// This is a rewrite of *casync* chunker (cachunker.h) in rust.
 ///
 /// Hashing by cyclic polynomial (also called Buzhash) has the benefit
 /// of avoiding multiplications, using barrel shifts instead. For more
 /// information please take a look at the [Rolling
-/// Hash](https://en.wikipedia.org/wiki/Rolling_hash) artikel from
-/// wikipedia.
+/// Hash](https://en.wikipedia.org/wiki/Rolling_hash) article from
+/// Wikipedia.
 
 pub struct Chunker {
     h: u32,

src/backup/crypt_config.rs

@@ -6,12 +6,30 @@
 //! See the Wikipedia Artikel for [Authenticated
 //! encryption](https://en.wikipedia.org/wiki/Authenticated_encryption)
 //! for a short introduction.
-use anyhow::{bail, Error};
-use openssl::pkcs5::pbkdf2_hmac;
-use openssl::hash::MessageDigest;
-use openssl::symm::{decrypt_aead, Cipher, Crypter, Mode};
+
 use std::io::Write;
+
+use anyhow::{bail, Error};
 use chrono::{Local, TimeZone, DateTime};
+use openssl::hash::MessageDigest;
+use openssl::pkcs5::pbkdf2_hmac;
+use openssl::symm::{decrypt_aead, Cipher, Crypter, Mode};
+use serde::{Deserialize, Serialize};
+
+use proxmox::api::api;
+
+#[api(default: "encrypt")]
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+/// Defines whether data is encrypted (using an AEAD cipher), only signed, or neither.
+pub enum CryptMode {
+    /// Don't encrypt.
+    None,
+    /// Encrypt.
+    Encrypt,
+    /// Only sign.
+    SignOnly,
+}
 
 /// Encryption Configuration with secret key
 ///
@@ -26,7 +44,6 @@ pub struct CryptConfig {
     id_pkey: openssl::pkey::PKey<openssl::pkey::Private>,
     // The private key used by the cipher.
     enc_key: [u8; 32],
-
 }
 
 impl CryptConfig {
@@ -63,10 +80,9 @@ impl CryptConfig {
     /// chunk digest values do not clash with values computed for
     /// other sectret keys.
     pub fn compute_digest(&self, data: &[u8]) -> [u8; 32] {
-        // FIXME: use HMAC-SHA256 instead??
         let mut hasher = openssl::sha::Sha256::new();
-        hasher.update(&self.id_key);
         hasher.update(data);
+        hasher.update(&self.id_key); // at the end, to avoid length extensions attacks
         hasher.finish()
     }
 
@@ -203,7 +219,7 @@ impl CryptConfig {
         created: DateTime<Local>,
     ) -> Result<Vec<u8>, Error> {
 
-         let modified = Local.timestamp(Local::now().timestamp(), 0);
+        let modified = Local.timestamp(Local::now().timestamp(), 0);
         let key_config = super::KeyConfig { kdf: None, created, modified, data: self.enc_key.to_vec() };
         let data = serde_json::to_string(&key_config)?.as_bytes().to_vec();
 

src/backup/data_blob.rs

@@ -3,10 +3,10 @@ use std::convert::TryInto;
 
 use proxmox::tools::io::{ReadExt, WriteExt};
 
-const MAX_BLOB_SIZE: usize = 128*1024*1024;
-
 use super::file_formats::*;
-use super::CryptConfig;
+use super::{CryptConfig, CryptMode};
+
+const MAX_BLOB_SIZE: usize = 128*1024*1024;
 
 /// Encoded data chunk with digest and positional information
 pub struct ChunkInfo {
@@ -36,6 +36,11 @@ impl DataBlob {
         &self.raw_data
     }
 
+    /// Returns raw_data size
+    pub fn raw_size(&self) -> u64 {
+        self.raw_data.len() as u64
+    }
+
     /// Consume self and returns raw_data
     pub fn into_inner(self) -> Vec<u8> {
         self.raw_data
@@ -66,8 +71,8 @@ impl DataBlob {
         hasher.finalize()
     }
 
-    /// verify the CRC32 checksum
-    pub fn verify_crc(&self) -> Result<(), Error> {
+    // verify the CRC32 checksum
+    fn verify_crc(&self) -> Result<(), Error> {
         let expected_crc = self.compute_crc();
         if expected_crc != self.crc() {
             bail!("Data blob has wrong CRC checksum.");
@@ -166,17 +171,37 @@ impl DataBlob {
         Ok(blob)
     }
 
+    /// Get the encryption mode for this blob.
+    pub fn crypt_mode(&self) -> Result<CryptMode, Error> {
+        let magic = self.magic();
+
+        Ok(if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 || magic == &COMPRESSED_BLOB_MAGIC_1_0 {
+            CryptMode::None
+        } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
+            CryptMode::Encrypt
+        } else {
+            bail!("Invalid blob magic number.");
+        })
+    }
+
     /// Decode blob data
-    pub fn decode(self, config: Option<&CryptConfig>) -> Result<Vec<u8>, Error> {
+    pub fn decode(&self, config: Option<&CryptConfig>, digest: Option<&[u8; 32]>) -> Result<Vec<u8>, Error> {
 
         let magic = self.magic();
 
         if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
             let data_start = std::mem::size_of::<DataBlobHeader>();
-            Ok(self.raw_data[data_start..].to_vec())
+            let data = self.raw_data[data_start..].to_vec();
+            if let Some(digest) = digest {
+                Self::verify_digest(&data, None, digest)?;
+            }
+            Ok(data)
         } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
             let data_start = std::mem::size_of::<DataBlobHeader>();
             let data = zstd::block::decompress(&self.raw_data[data_start..], MAX_BLOB_SIZE)?;
+            if let Some(digest) = digest {
+                Self::verify_digest(&data, None, digest)?;
+            }
             Ok(data)
         } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
             let header_len = std::mem::size_of::<EncryptedDataBlobHeader>();
@@ -190,86 +215,29 @@ impl DataBlob {
                 } else {
                     config.decode_uncompressed_chunk(&self.raw_data[header_len..], &head.iv, &head.tag)?
                 };
+                if let Some(digest) = digest {
+                    Self::verify_digest(&data, Some(config), digest)?;
+                }
                 Ok(data)
             } else {
                 bail!("unable to decrypt blob - missing CryptConfig");
             }
-        } else if magic == &AUTH_COMPR_BLOB_MAGIC_1_0 || magic == &AUTHENTICATED_BLOB_MAGIC_1_0 {
-            let header_len = std::mem::size_of::<AuthenticatedDataBlobHeader>();
-            let head = unsafe {
-                (&self.raw_data[..header_len]).read_le_value::<AuthenticatedDataBlobHeader>()?
-            };
-
-            let data_start = std::mem::size_of::<AuthenticatedDataBlobHeader>();
-
-            // Note: only verify if we have a crypt config
-            if let Some(config) = config  {
-                let signature = config.compute_auth_tag(&self.raw_data[data_start..]);
-                if signature != head.tag {
-                    bail!("verifying blob signature failed");
-                }
-            }
-
-            if magic == &AUTH_COMPR_BLOB_MAGIC_1_0 {
-                let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?;
-                Ok(data)
-            } else {
-                Ok(self.raw_data[data_start..].to_vec())
-            }
         } else {
             bail!("Invalid blob magic number.");
         }
     }
 
-    /// Create a signed DataBlob, optionally compressed
-    pub fn create_signed(
-        data: &[u8],
-        config: &CryptConfig,
-        compress: bool,
-    ) -> Result<Self, Error> {
-
-        if data.len() > MAX_BLOB_SIZE {
-            bail!("data blob too large ({} bytes).", data.len());
-        }
-
-        let compr_data;
-        let (_compress, data, magic) = if compress {
-            compr_data = zstd::block::compress(data, 1)?;
-            // Note: We only use compression if result is shorter
-            if compr_data.len() < data.len() {
-                (true, &compr_data[..], AUTH_COMPR_BLOB_MAGIC_1_0)
-            } else {
-                (false, data, AUTHENTICATED_BLOB_MAGIC_1_0)
-            }
-        } else {
-            (false, data, AUTHENTICATED_BLOB_MAGIC_1_0)
-        };
-
-        let header_len = std::mem::size_of::<AuthenticatedDataBlobHeader>();
-        let mut raw_data = Vec::with_capacity(data.len() + header_len);
-
-        let head = AuthenticatedDataBlobHeader {
-            head: DataBlobHeader { magic, crc: [0; 4] },
-            tag: config.compute_auth_tag(data),
-        };
-        unsafe {
-            raw_data.write_le_value(head)?;
-        }
-        raw_data.extend_from_slice(data);
-
-        let mut blob = DataBlob { raw_data };
-        blob.set_crc(blob.compute_crc());
-
-        Ok(blob)
-    }
-
-    /// Load blob from ``reader``
-    pub fn load(reader: &mut dyn std::io::Read) -> Result<Self, Error> {
+    /// Load blob from ``reader``, verify CRC
+    pub fn load_from_reader(reader: &mut dyn std::io::Read) -> Result<Self, Error> {
 
         let mut data = Vec::with_capacity(1024*1024);
         reader.read_to_end(&mut data)?;
 
-        Self::from_raw(data)
+        let blob = Self::from_raw(data)?;
+
+        blob.verify_crc()?;
+
+        Ok(blob)
     }
 
     /// Create Instance from raw data
@@ -294,14 +262,6 @@ impl DataBlob {
 
             let blob = DataBlob { raw_data: data };
 
-            Ok(blob)
-        } else if magic == AUTH_COMPR_BLOB_MAGIC_1_0 || magic == AUTHENTICATED_BLOB_MAGIC_1_0 {
-            if data.len() < std::mem::size_of::<AuthenticatedDataBlobHeader>() {
-                bail!("authenticated blob too small ({} bytes).", data.len());
-            }
-
-            let blob = DataBlob { raw_data: data };
-
             Ok(blob)
         } else {
             bail!("unable to parse raw blob - wrong magic");
@@ -311,7 +271,9 @@ impl DataBlob {
     /// Verify digest and data length for unencrypted chunks.
     ///
     /// To do that, we need to decompress data first. Please note that
-    /// this is noth possible for encrypted chunks.
+    /// this is not possible for encrypted chunks. This function simply return Ok
+    /// for encrypted chunks.
+    /// Note: This does not call verify_crc, because this is usually done in load
     pub fn verify_unencrypted(
         &self,
         expected_chunk_size: usize,
@@ -320,22 +282,32 @@ impl DataBlob {
 
         let magic = self.magic();
 
-        let verify_raw_data = |data: &[u8]| {
-            if expected_chunk_size != data.len() {
-                bail!("detected chunk with wrong length ({} != {})", expected_chunk_size, data.len());
-            }
-            let digest = openssl::sha::sha256(data);
-            if &digest != expected_digest {
-                bail!("detected chunk with wrong digest.");
-            }
-            Ok(())
-        };
+        if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
+            return Ok(());
+        }
 
-        if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
-            let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
-            verify_raw_data(&data)?;
-        } else if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
-            verify_raw_data(&self.raw_data[12..])?;
+        // verifies digest!
+        let data = self.decode(None, Some(expected_digest))?;
+
+        if expected_chunk_size != data.len() {
+            bail!("detected chunk with wrong length ({} != {})", expected_chunk_size, data.len());
+        }
+
+        Ok(())
+    }
+
+    fn verify_digest(
+        data: &[u8],
+        config: Option<&CryptConfig>,
+        expected_digest: &[u8; 32],
+    ) -> Result<(), Error> {
+
+        let digest = match config {
+            Some(config) => config.compute_digest(data),
+            None => openssl::sha::sha256(&data),
+        };
+        if &digest != expected_digest {
+            bail!("detected chunk with wrong digest.");
         }
 
         Ok(())
@@ -378,7 +350,7 @@ impl <'a, 'b> DataChunkBuilder<'a, 'b> {
 
     /// Set encryption Configuration
     ///
-    /// If set, chunks are encrypted.
+    /// If set, chunks are encrypted
     pub fn crypt_config(mut self, value: &'b CryptConfig) -> Self {
         if self.digest_computed {
             panic!("unable to set crypt_config after compute_digest().");
@@ -417,12 +389,7 @@ impl <'a, 'b> DataChunkBuilder<'a, 'b> {
             self.compute_digest();
         }
 
-        let chunk = DataBlob::encode(
-            self.orig_data,
-            self.config,
-            self.compress,
-        )?;
-
+        let chunk = DataBlob::encode(self.orig_data, self.config, self.compress)?;
         Ok((chunk, self.digest))
     }
 

src/backup/data_blob_reader.rs

@@ -1,4 +1,4 @@
-use anyhow::{bail, Error};
+use anyhow::{bail, format_err, Error};
 use std::sync::Arc;
 use std::io::{Read, BufReader};
 use proxmox::tools::io::ReadExt;
@@ -8,8 +8,6 @@ use super::*;
 enum BlobReaderState<R: Read> {
     Uncompressed { expected_crc: u32, csum_reader: ChecksumReader<R> },
     Compressed { expected_crc: u32, decompr: zstd::stream::read::Decoder<BufReader<ChecksumReader<R>>> },
-    Signed { expected_crc: u32, expected_hmac: [u8; 32], csum_reader: ChecksumReader<R> },
-    SignedCompressed { expected_crc: u32, expected_hmac: [u8; 32], decompr: zstd::stream::read::Decoder<BufReader<ChecksumReader<R>>> },
     Encrypted { expected_crc: u32, decrypt_reader: CryptReader<BufReader<ChecksumReader<R>>> },
     EncryptedCompressed { expected_crc: u32, decompr: zstd::stream::read::Decoder<BufReader<CryptReader<BufReader<ChecksumReader<R>>>>> },
 }
@@ -19,6 +17,10 @@ pub struct DataBlobReader<R: Read> {
     state: BlobReaderState<R>,
 }
 
+// zstd_safe::DCtx is not sync but we are, since
+// the only public interface is on mutable reference
+unsafe impl<R: Read> Sync for DataBlobReader<R> {}
+
 impl <R: Read> DataBlobReader<R> {
 
     pub fn new(mut reader: R, config: Option<Arc<CryptConfig>>) -> Result<Self, Error> {
@@ -37,40 +39,26 @@ impl <R: Read> DataBlobReader<R> {
                 let decompr = zstd::stream::read::Decoder::new(csum_reader)?;
                 Ok(Self { state: BlobReaderState::Compressed { expected_crc, decompr }})
             }
-            AUTHENTICATED_BLOB_MAGIC_1_0 => {
-                let expected_crc = u32::from_le_bytes(head.crc);
-                let mut expected_hmac = [0u8; 32];
-                reader.read_exact(&mut expected_hmac)?;
-                let csum_reader = ChecksumReader::new(reader, config);
-                Ok(Self { state: BlobReaderState::Signed { expected_crc, expected_hmac, csum_reader }})
-            }
-            AUTH_COMPR_BLOB_MAGIC_1_0 => {
-                let expected_crc = u32::from_le_bytes(head.crc);
-                let mut expected_hmac = [0u8; 32];
-                reader.read_exact(&mut expected_hmac)?;
-                let csum_reader = ChecksumReader::new(reader, config);
-
-                let decompr = zstd::stream::read::Decoder::new(csum_reader)?;
-                Ok(Self { state: BlobReaderState::SignedCompressed { expected_crc, expected_hmac, decompr }})
-            }
             ENCRYPTED_BLOB_MAGIC_1_0 => {
+                let config = config.ok_or_else(|| format_err!("unable to read encrypted blob without key"))?;
                 let expected_crc = u32::from_le_bytes(head.crc);
                 let mut iv = [0u8; 16];
                 let mut expected_tag = [0u8; 16];
                 reader.read_exact(&mut iv)?;
                 reader.read_exact(&mut expected_tag)?;
                 let csum_reader = ChecksumReader::new(reader, None);
-                let decrypt_reader = CryptReader::new(BufReader::with_capacity(64*1024, csum_reader), iv, expected_tag, config.unwrap())?;
+                let decrypt_reader = CryptReader::new(BufReader::with_capacity(64*1024, csum_reader), iv, expected_tag, config)?;
                 Ok(Self { state: BlobReaderState::Encrypted { expected_crc, decrypt_reader }})
             }
             ENCR_COMPR_BLOB_MAGIC_1_0 => {
+                let config = config.ok_or_else(|| format_err!("unable to read encrypted blob without key"))?;
                 let expected_crc = u32::from_le_bytes(head.crc);
                 let mut iv = [0u8; 16];
                 let mut expected_tag = [0u8; 16];
                 reader.read_exact(&mut iv)?;
                 reader.read_exact(&mut expected_tag)?;
                 let csum_reader = ChecksumReader::new(reader, None);
-                let decrypt_reader = CryptReader::new(BufReader::with_capacity(64*1024, csum_reader), iv, expected_tag, config.unwrap())?;
+                let decrypt_reader = CryptReader::new(BufReader::with_capacity(64*1024, csum_reader), iv, expected_tag, config)?;
                 let decompr = zstd::stream::read::Decoder::new(decrypt_reader)?;
                 Ok(Self { state: BlobReaderState::EncryptedCompressed { expected_crc, decompr }})
             }
@@ -95,31 +83,6 @@ impl <R: Read> DataBlobReader<R> {
                 }
                 Ok(reader)
             }
-            BlobReaderState::Signed { csum_reader, expected_crc, expected_hmac } => {
-                let (reader, crc, hmac) = csum_reader.finish()?;
-                if crc != expected_crc {
-                    bail!("blob crc check failed");
-                }
-                if let Some(hmac) = hmac {
-                    if hmac != expected_hmac {
-                        bail!("blob signature check failed");
-                    }
-                }
-                Ok(reader)
-            }
-            BlobReaderState::SignedCompressed { expected_crc, expected_hmac, decompr } => {
-                let csum_reader = decompr.finish().into_inner();
-                let (reader, crc, hmac) = csum_reader.finish()?;
-                if crc != expected_crc {
-                    bail!("blob crc check failed");
-                }
-                if let Some(hmac) = hmac {
-                    if hmac != expected_hmac {
-                        bail!("blob signature check failed");
-                    }
-                }
-                Ok(reader)
-            }
             BlobReaderState::Encrypted { expected_crc, decrypt_reader } =>  {
                 let csum_reader = decrypt_reader.finish()?.into_inner();
                 let (reader, crc, _) = csum_reader.finish()?;
@@ -151,12 +114,6 @@ impl <R: Read> Read for DataBlobReader<R> {
             BlobReaderState::Compressed { decompr, .. } => {
                 decompr.read(buf)
             }
-            BlobReaderState::Signed { csum_reader, .. } => {
-                csum_reader.read(buf)
-            }
-            BlobReaderState::SignedCompressed { decompr, .. } => {
-                decompr.read(buf)
-            }
             BlobReaderState::Encrypted { decrypt_reader, .. } =>  {
                 decrypt_reader.read(buf)
             }

src/backup/data_blob_writer.rs

@@ -8,8 +8,6 @@ use super::*;
 enum BlobWriterState<W: Write> {
     Uncompressed { csum_writer: ChecksumWriter<W> },
     Compressed { compr: zstd::stream::write::Encoder<ChecksumWriter<W>> },
-    Signed { csum_writer: ChecksumWriter<W> },
-    SignedCompressed { compr: zstd::stream::write::Encoder<ChecksumWriter<W>> },
     Encrypted { crypt_writer: CryptWriter<ChecksumWriter<W>> },
     EncryptedCompressed { compr: zstd::stream::write::Encoder<CryptWriter<ChecksumWriter<W>>> },
 }
@@ -42,33 +40,6 @@ impl <W: Write + Seek> DataBlobWriter<W> {
         Ok(Self { state: BlobWriterState::Compressed { compr }})
     }
 
-    pub fn new_signed(mut writer: W, config: Arc<CryptConfig>) -> Result<Self, Error> {
-        writer.seek(SeekFrom::Start(0))?;
-        let head = AuthenticatedDataBlobHeader {
-            head: DataBlobHeader { magic: AUTHENTICATED_BLOB_MAGIC_1_0, crc: [0; 4] },
-            tag: [0u8; 32],
-        };
-        unsafe {
-            writer.write_le_value(head)?;
-        }
-        let csum_writer = ChecksumWriter::new(writer, Some(config));
-        Ok(Self { state:  BlobWriterState::Signed { csum_writer }})
-    }
-
-    pub fn new_signed_compressed(mut writer: W, config: Arc<CryptConfig>) -> Result<Self, Error> {
-        writer.seek(SeekFrom::Start(0))?;
-        let head = AuthenticatedDataBlobHeader {
-            head: DataBlobHeader { magic: AUTH_COMPR_BLOB_MAGIC_1_0, crc: [0; 4] },
-            tag: [0u8; 32],
-        };
-        unsafe {
-            writer.write_le_value(head)?;
-        }
-        let csum_writer = ChecksumWriter::new(writer, Some(config));
-        let compr = zstd::stream::write::Encoder::new(csum_writer, 1)?;
-        Ok(Self { state: BlobWriterState::SignedCompressed { compr }})
-    }
-
     pub fn new_encrypted(mut writer: W, config: Arc<CryptConfig>) -> Result<Self, Error> {
         writer.seek(SeekFrom::Start(0))?;
         let head = EncryptedDataBlobHeader {
@@ -129,37 +100,6 @@ impl <W: Write + Seek> DataBlobWriter<W> {
 
                 Ok(writer)
             }
-            BlobWriterState::Signed { csum_writer } => {
-                let (mut writer, crc, tag) = csum_writer.finish()?;
-
-                let head = AuthenticatedDataBlobHeader {
-                    head: DataBlobHeader { magic: AUTHENTICATED_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() },
-                    tag: tag.unwrap(),
-                };
-
-                writer.seek(SeekFrom::Start(0))?;
-                unsafe {
-                    writer.write_le_value(head)?;
-                }
-
-                Ok(writer)
-            }
-            BlobWriterState::SignedCompressed { compr } => {
-                let csum_writer = compr.finish()?;
-                let (mut writer, crc, tag) = csum_writer.finish()?;
-
-                let head = AuthenticatedDataBlobHeader {
-                    head: DataBlobHeader { magic: AUTH_COMPR_BLOB_MAGIC_1_0, crc: crc.to_le_bytes() },
-                    tag: tag.unwrap(),
-                };
-
-                writer.seek(SeekFrom::Start(0))?;
-                unsafe {
-                    writer.write_le_value(head)?;
-                }
-
-                Ok(writer)
-            }
             BlobWriterState::Encrypted { crypt_writer } => {
                 let (csum_writer, iv, tag) = crypt_writer.finish()?;
                 let (mut writer, crc, _) = csum_writer.finish()?;
@@ -203,12 +143,6 @@ impl <W: Write + Seek> Write for DataBlobWriter<W> {
             BlobWriterState::Compressed { ref mut compr } => {
                 compr.write(buf)
             }
-            BlobWriterState::Signed { ref mut csum_writer } => {
-                csum_writer.write(buf)
-            }
-            BlobWriterState::SignedCompressed { ref mut compr } => {
-               compr.write(buf)
-            }
             BlobWriterState::Encrypted { ref mut crypt_writer } => {
                 crypt_writer.write(buf)
             }
@@ -226,13 +160,7 @@ impl <W: Write + Seek> Write for DataBlobWriter<W> {
             BlobWriterState::Compressed { ref mut compr } => {
                 compr.flush()
             }
-            BlobWriterState::Signed { ref mut csum_writer } => {
-                csum_writer.flush()
-            }
-            BlobWriterState::SignedCompressed { ref mut compr } => {
-                compr.flush()
-            }
-            BlobWriterState::Encrypted { ref mut crypt_writer } => {
+             BlobWriterState::Encrypted { ref mut crypt_writer } => {
                crypt_writer.flush()
             }
             BlobWriterState::EncryptedCompressed { ref mut compr } => {

src/backup/datastore.rs

@@ -2,22 +2,28 @@ use std::collections::{HashSet, HashMap};
 use std::io::{self, Write};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
+use std::convert::TryFrom;
 
 use anyhow::{bail, format_err, Error};
 use lazy_static::lazy_static;
 use chrono::{DateTime, Utc};
+use serde_json::Value;
+
+use proxmox::tools::fs::{replace_file, CreateOptions};
 
 use super::backup_info::{BackupGroup, BackupDir};
 use super::chunk_store::ChunkStore;
 use super::dynamic_index::{DynamicIndexReader, DynamicIndexWriter};
 use super::fixed_index::{FixedIndexReader, FixedIndexWriter};
-use super::manifest::{MANIFEST_BLOB_NAME, BackupManifest};
+use super::manifest::{MANIFEST_BLOB_NAME, CLIENT_LOG_BLOB_NAME, BackupManifest};
 use super::index::*;
 use super::{DataBlob, ArchiveType, archive_type};
 use crate::config::datastore;
 use crate::server::WorkerTask;
 use crate::tools;
-use crate::api2::types::GarbageCollectionStatus;
+use crate::tools::format::HumanByte;
+use crate::tools::fs::{lock_dir_noblock, DirLockGuard};
+use crate::api2::types::{GarbageCollectionStatus, Userid};
 
 lazy_static! {
     static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new());
@@ -134,11 +140,15 @@ impl DataStore {
         Ok(out)
     }
 
+    pub fn name(&self) -> &str {
+        self.chunk_store.name()
+    }
+
     pub fn base_path(&self) -> PathBuf {
         self.chunk_store.base_path()
     }
 
-    /// Clenaup a backup directory
+    /// Cleanup a backup directory
     ///
     /// Removes all files not mentioned in the manifest.
     pub fn cleanup_backup_dir(&self, backup_dir: &BackupDir, manifest: &BackupManifest
@@ -149,6 +159,7 @@ impl DataStore {
 
         let mut wanted_files = HashSet::new();
         wanted_files.insert(MANIFEST_BLOB_NAME.to_string());
+        wanted_files.insert(CLIENT_LOG_BLOB_NAME.to_string());
         manifest.files().iter().for_each(|item| { wanted_files.insert(item.filename.clone()); });
 
         for item in tools::fs::read_subdir(libc::AT_FDCWD, &full_path)? {
@@ -190,6 +201,8 @@ impl DataStore {
 
         let full_path = self.group_path(backup_group);
 
+        let _guard = tools::fs::lock_dir_noblock(&full_path, "backup group", "possible running backup")?;
+
         log::info!("removing backup group {:?}", full_path);
         std::fs::remove_dir_all(&full_path)
             .map_err(|err| {
@@ -204,10 +217,15 @@ impl DataStore {
     }
 
     /// Remove a backup directory including all content
-    pub fn remove_backup_dir(&self, backup_dir: &BackupDir) ->  Result<(), Error> {
+    pub fn remove_backup_dir(&self, backup_dir: &BackupDir, force: bool) ->  Result<(), Error> {
 
         let full_path = self.snapshot_path(backup_dir);
 
+        let _guard;
+        if !force {
+            _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or used as base")?;
+        }
+
         log::info!("removing backup snapshot {:?}", full_path);
         std::fs::remove_dir_all(&full_path)
             .map_err(|err| {
@@ -239,16 +257,21 @@ impl DataStore {
     /// Returns the backup owner.
     ///
     /// The backup owner is the user who first created the backup group.
-    pub fn get_owner(&self, backup_group: &BackupGroup) -> Result<String, Error> {
+    pub fn get_owner(&self, backup_group: &BackupGroup) -> Result<Userid, Error> {
         let mut full_path = self.base_path();
         full_path.push(backup_group.group_path());
         full_path.push("owner");
         let owner = proxmox::tools::fs::file_read_firstline(full_path)?;
-        Ok(owner.trim_end().to_string()) // remove trailing newline
+        Ok(owner.trim_end().parse()?) // remove trailing newline
     }
 
     /// Set the backup owner.
-    pub fn set_owner(&self, backup_group: &BackupGroup, userid: &str, force: bool) -> Result<(), Error> {
+    pub fn set_owner(
+        &self,
+        backup_group: &BackupGroup,
+        userid: &Userid,
+        force: bool,
+    ) -> Result<(), Error> {
         let mut path = self.base_path();
         path.push(backup_group.group_path());
         path.push("owner");
@@ -272,12 +295,17 @@ impl DataStore {
         Ok(())
     }
 
-    /// Create a backup group if it does not already exists.
+    /// Create (if it does not already exists) and lock a backup group
     ///
     /// And set the owner to 'userid'. If the group already exists, it returns the
     /// current owner (instead of setting the owner).
-    pub fn create_backup_group(&self, backup_group: &BackupGroup, userid: &str) -> Result<String, Error> {
-
+    ///
+    /// This also acquires an exclusive lock on the directory and returns the lock guard.
+    pub fn create_locked_backup_group(
+        &self,
+        backup_group: &BackupGroup,
+        userid: &Userid,
+    ) -> Result<(Userid, DirLockGuard), Error> {
         // create intermediate path first:
         let base_path = self.base_path();
 
@@ -290,13 +318,15 @@ impl DataStore {
         // create the last component now
         match std::fs::create_dir(&full_path) {
             Ok(_) => {
+                let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?;
                 self.set_owner(backup_group, userid, false)?;
                 let owner = self.get_owner(backup_group)?; // just to be sure
-                Ok(owner)
+                Ok((owner, guard))
             }
             Err(ref err) if err.kind() == io::ErrorKind::AlreadyExists => {
+                let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?;
                 let owner = self.get_owner(backup_group)?; // just to be sure
-                Ok(owner)
+                Ok((owner, guard))
             }
             Err(err) => bail!("unable to create backup group {:?} - {}", full_path, err),
         }
@@ -305,15 +335,20 @@ impl DataStore {
     /// Creates a new backup snapshot inside a BackupGroup
     ///
     /// The BackupGroup directory needs to exist.
-    pub fn create_backup_dir(&self, backup_dir: &BackupDir) ->  Result<(PathBuf, bool), io::Error> {
+    pub fn create_locked_backup_dir(&self, backup_dir: &BackupDir)
+        -> Result<(PathBuf, bool, DirLockGuard), Error>
+    {
         let relative_path = backup_dir.relative_path();
         let mut full_path = self.base_path();
         full_path.push(&relative_path);
 
+        let lock = ||
+            lock_dir_noblock(&full_path, "snapshot", "internal error - tried creating snapshot that's already in use");
+
         match std::fs::create_dir(&full_path) {
-            Ok(_) => Ok((relative_path, true)),
-            Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => Ok((relative_path, false)),
-            Err(e) => Err(e)
+            Ok(_) => Ok((relative_path, true, lock()?)),
+            Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => Ok((relative_path, false, lock()?)),
+            Err(e) => Err(e.into())
         }
     }
 
@@ -333,9 +368,30 @@ impl DataStore {
                 .map(|s| s.starts_with("."))
                 .unwrap_or(false)
         }
-
+        let handle_entry_err = |err: walkdir::Error| {
+            if let Some(inner) = err.io_error() {
+                let path = err.path().unwrap_or(Path::new(""));
+                match inner.kind() {
+                    io::ErrorKind::PermissionDenied => {
+                        // only allow to skip ext4 fsck directory, avoid GC if, for example,
+                        // a user got file permissions wrong on datastore rsync to new server
+                        if err.depth() > 1 || !path.ends_with("lost+found") {
+                            bail!("cannot continue garbage-collection safely, permission denied on: {}", path.display())
+                        }
+                    },
+                    _ => bail!("unexpected error on datastore traversal: {} - {}", inner, path.display()),
+                }
+            }
+            Ok(())
+        };
         for entry in walker.filter_entry(|e| !is_hidden(e)) {
-            let path = entry?.into_path();
+            let path = match entry {
+                Ok(entry) => entry.into_path(),
+                Err(err) => {
+                    handle_entry_err(err)?;
+                    continue
+                },
+            };
             if let Ok(archive_type) = archive_type(&path) {
                 if archive_type == ArchiveType::FixedIndex || archive_type == ArchiveType::DynamicIndex {
                     list.push(path);
@@ -363,8 +419,8 @@ impl DataStore {
             tools::fail_on_shutdown()?;
             let digest = index.index_digest(pos).unwrap();
             if let Err(err) = self.chunk_store.touch_chunk(digest) {
-                bail!("unable to access chunk {}, required by {:?} - {}",
-                      proxmox::tools::digest_to_hex(digest), file_name, err);
+                worker.warn(&format!("warning: unable to access chunk {}, required by {:?} - {}",
+                      proxmox::tools::digest_to_hex(digest), file_name, err));
             }
         }
         Ok(())
@@ -407,9 +463,8 @@ impl DataStore {
 
             let _exclusive_lock =  self.chunk_store.try_exclusive_lock()?;
 
-            let now = unsafe { libc::time(std::ptr::null_mut()) };
-
-            let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(now);
+            let phase1_start_time = unsafe { libc::time(std::ptr::null_mut()) };
+            let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(phase1_start_time);
 
             let mut gc_status = GarbageCollectionStatus::default();
             gc_status.upid = Some(worker.to_string());
@@ -419,26 +474,26 @@ impl DataStore {
             self.mark_used_chunks(&mut gc_status, &worker)?;
 
             worker.log("Start GC phase2 (sweep unused chunks)");
-            self.chunk_store.sweep_unused_chunks(oldest_writer, &mut gc_status, &worker)?;
+            self.chunk_store.sweep_unused_chunks(oldest_writer, phase1_start_time, &mut gc_status, &worker)?;
 
-            worker.log(&format!("Removed bytes: {}", gc_status.removed_bytes));
+            worker.log(&format!("Removed garbage: {}", HumanByte::from(gc_status.removed_bytes)));
             worker.log(&format!("Removed chunks: {}", gc_status.removed_chunks));
             if gc_status.pending_bytes > 0 {
-                worker.log(&format!("Pending removals: {} bytes ({} chunks)", gc_status.pending_bytes, gc_status.pending_chunks));
+                worker.log(&format!("Pending removals: {} (in {} chunks)", HumanByte::from(gc_status.pending_bytes), gc_status.pending_chunks));
             }
 
-            worker.log(&format!("Original data bytes: {}", gc_status.index_data_bytes));
+            worker.log(&format!("Original data usage: {}", HumanByte::from(gc_status.index_data_bytes)));
 
             if gc_status.index_data_bytes > 0 {
-                let comp_per = (gc_status.disk_bytes*100)/gc_status.index_data_bytes;
-                worker.log(&format!("Disk bytes: {} ({} %)", gc_status.disk_bytes, comp_per));
+                let comp_per = (gc_status.disk_bytes as f64 * 100.)/gc_status.index_data_bytes as f64;
+                worker.log(&format!("On-Disk usage: {} ({:.2}%)", HumanByte::from(gc_status.disk_bytes), comp_per));
             }
 
-            worker.log(&format!("Disk chunks: {}", gc_status.disk_chunks));
+            worker.log(&format!("On-Disk chunks: {}", gc_status.disk_chunks));
 
             if gc_status.disk_chunks > 0 {
                 let avg_chunk = gc_status.disk_bytes/(gc_status.disk_chunks as u64);
-                worker.log(&format!("Average chunk size: {}", avg_chunk));
+                worker.log(&format!("Average chunk size: {}", HumanByte::from(avg_chunk)));
             }
 
             *self.last_gc_status.lock().unwrap() = gc_status;
@@ -469,4 +524,70 @@ impl DataStore {
     ) -> Result<(bool, u64), Error> {
         self.chunk_store.insert_chunk(chunk, digest)
     }
+
+    pub fn load_blob(&self, backup_dir: &BackupDir, filename: &str) -> Result<DataBlob, Error> {
+        let mut path = self.base_path();
+        path.push(backup_dir.relative_path());
+        path.push(filename);
+
+        proxmox::try_block!({
+            let mut file = std::fs::File::open(&path)?;
+            DataBlob::load_from_reader(&mut file)
+        }).map_err(|err| format_err!("unable to load blob '{:?}' - {}", path, err))
+    }
+
+
+    pub fn load_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> {
+
+        let (chunk_path, digest_str) = self.chunk_store.chunk_path(digest);
+
+        proxmox::try_block!({
+            let mut file = std::fs::File::open(&chunk_path)?;
+            DataBlob::load_from_reader(&mut file)
+        }).map_err(|err| format_err!(
+            "store '{}', unable to load chunk '{}' - {}",
+            self.name(),
+            digest_str,
+            err,
+        ))
+     }
+
+    pub fn load_manifest(
+        &self,
+        backup_dir: &BackupDir,
+    ) -> Result<(BackupManifest, u64), Error> {
+        let blob = self.load_blob(backup_dir, MANIFEST_BLOB_NAME)?;
+        let raw_size = blob.raw_size();
+        let manifest = BackupManifest::try_from(blob)?;
+        Ok((manifest, raw_size))
+    }
+
+    pub fn load_manifest_json(
+        &self,
+        backup_dir: &BackupDir,
+    ) -> Result<Value, Error> {
+        let blob = self.load_blob(backup_dir, MANIFEST_BLOB_NAME)?;
+        // no expected digest available
+        let manifest_data = blob.decode(None, None)?;
+        let manifest: Value = serde_json::from_slice(&manifest_data[..])?;
+        Ok(manifest)
+    }
+
+    pub fn store_manifest(
+        &self,
+        backup_dir: &BackupDir,
+        manifest: Value,
+    ) -> Result<(), Error> {
+        let manifest = serde_json::to_string_pretty(&manifest)?;
+        let blob = DataBlob::encode(manifest.as_bytes(), None, true)?;
+        let raw_data = blob.raw_data();
+
+        let mut path = self.base_path();
+        path.push(backup_dir.relative_path());
+        path.push(MANIFEST_BLOB_NAME);
+
+        replace_file(&path, raw_data, CreateOptions::new())?;
+
+        Ok(())
+    }
 }

src/backup/dynamic_index.rs

@@ -1,23 +1,27 @@
-use std::convert::TryInto;
 use std::fs::File;
-use std::io::{BufWriter, Seek, SeekFrom, Write};
+use std::io::{self, BufWriter, Seek, SeekFrom, Write};
+use std::ops::Range;
 use std::os::unix::io::AsRawFd;
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
+use std::task::Context;
+use std::pin::Pin;
 
 use anyhow::{bail, format_err, Error};
 
 use proxmox::tools::io::ReadExt;
 use proxmox::tools::uuid::Uuid;
-use proxmox::tools::vec;
+use proxmox::tools::mmap::Mmap;
+use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
 
 use super::chunk_stat::ChunkStat;
 use super::chunk_store::ChunkStore;
+use super::index::ChunkReadInfo;
 use super::read_chunk::ReadChunk;
 use super::Chunker;
 use super::IndexFile;
 use super::{DataBlob, DataChunkBuilder};
-use crate::tools;
+use crate::tools::{self, epoch_now_u64};
 
 /// Header format definition for dynamic index files (`.dixd`)
 #[repr(C)]
@@ -36,34 +40,52 @@ proxmox::static_assert_size!(DynamicIndexHeader, 4096);
 //     pub data: DynamicIndexHeaderData,
 // }
 
+impl DynamicIndexHeader {
+    /// Convenience method to allocate a zero-initialized header struct.
+    pub fn zeroed() -> Box<Self> {
+        unsafe {
+            Box::from_raw(std::alloc::alloc_zeroed(std::alloc::Layout::new::<Self>()) as *mut Self)
+        }
+    }
+
+    pub fn as_bytes(&self) -> &[u8] {
+        unsafe {
+            std::slice::from_raw_parts(
+                self as *const Self as *const u8,
+                std::mem::size_of::<Self>(),
+            )
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+#[repr(C)]
+pub struct DynamicEntry {
+    end_le: u64,
+    digest: [u8; 32],
+}
+
+impl DynamicEntry {
+    #[inline]
+    pub fn end(&self) -> u64 {
+        u64::from_le(self.end_le)
+    }
+}
+
 pub struct DynamicIndexReader {
     _file: File,
     pub size: usize,
-    index: *const u8,
-    index_entries: usize,
+    index: Mmap<DynamicEntry>,
     pub uuid: [u8; 16],
     pub ctime: u64,
     pub index_csum: [u8; 32],
 }
 
-// `index` is mmap()ed which cannot be thread-local so should be sendable
-// FIXME: Introduce an mmap wrapper type for this?
-unsafe impl Send for DynamicIndexReader {}
-unsafe impl Sync for DynamicIndexReader {}
-
-impl Drop for DynamicIndexReader {
-    fn drop(&mut self) {
-        if let Err(err) = self.unmap() {
-            eprintln!("Unable to unmap dynamic index - {}", err);
-        }
-    }
-}
-
 impl DynamicIndexReader {
     pub fn open(path: &Path) -> Result<Self, Error> {
         File::open(path)
             .map_err(Error::from)
-            .and_then(|file| Self::new(file))
+            .and_then(Self::new)
             .map_err(|err| format_err!("Unable to open dynamic index {:?} - {}", path, err))
     }
 
@@ -74,6 +96,7 @@ impl DynamicIndexReader {
             bail!("unable to get shared lock - {}", err);
         }
 
+        // FIXME: This is NOT OUR job! Check the callers of this method and remove this!
         file.seek(SeekFrom::Start(0))?;
 
         let header_size = std::mem::size_of::<DynamicIndexHeader>();
@@ -93,123 +116,49 @@ impl DynamicIndexReader {
         let size = stat.st_size as usize;
 
         let index_size = size - header_size;
-        if (index_size % 40) != 0 {
+        let index_count = index_size / 40;
+        if index_count * 40 != index_size {
             bail!("got unexpected file size");
         }
 
-        let data = unsafe {
-            nix::sys::mman::mmap(
-                std::ptr::null_mut(),
-                index_size,
+        let index = unsafe {
+            Mmap::map_fd(
+                rawfd,
+                header_size as u64,
+                index_count,
                 nix::sys::mman::ProtFlags::PROT_READ,
                 nix::sys::mman::MapFlags::MAP_PRIVATE,
-                rawfd,
-                header_size as i64,
-            )
-        }? as *const u8;
+            )?
+        };
 
         Ok(Self {
             _file: file,
             size,
-            index: data,
-            index_entries: index_size / 40,
+            index,
             ctime,
             uuid: header.uuid,
             index_csum: header.index_csum,
         })
     }
 
-    fn unmap(&mut self) -> Result<(), Error> {
-        if self.index == std::ptr::null_mut() {
-            return Ok(());
-        }
-
-        if let Err(err) = unsafe {
-            nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, self.index_entries * 40)
-        } {
-            bail!("unmap dynamic index failed - {}", err);
-        }
-
-        self.index = std::ptr::null_mut();
-
-        Ok(())
-    }
-
-    #[allow(clippy::cast_ptr_alignment)]
-    pub fn chunk_info(&self, pos: usize) -> Result<(u64, u64, [u8; 32]), Error> {
-        if pos >= self.index_entries {
-            bail!("chunk index out of range");
-        }
-        let start = if pos == 0 {
-            0
-        } else {
-            unsafe { *(self.index.add((pos - 1) * 40) as *const u64) }
-        };
-
-        let end = unsafe { *(self.index.add(pos * 40) as *const u64) };
-
-        let mut digest = std::mem::MaybeUninit::<[u8; 32]>::uninit();
-        unsafe {
-            std::ptr::copy_nonoverlapping(
-                self.index.add(pos * 40 + 8),
-                (*digest.as_mut_ptr()).as_mut_ptr(),
-                32,
-            );
-        }
-
-        Ok((start, end, unsafe { digest.assume_init() }))
-    }
-
     #[inline]
     #[allow(clippy::cast_ptr_alignment)]
     fn chunk_end(&self, pos: usize) -> u64 {
-        if pos >= self.index_entries {
+        if pos >= self.index.len() {
             panic!("chunk index out of range");
         }
-        unsafe { *(self.index.add(pos * 40) as *const u64) }
+        self.index[pos].end()
     }
 
     #[inline]
     fn chunk_digest(&self, pos: usize) -> &[u8; 32] {
-        if pos >= self.index_entries {
+        if pos >= self.index.len() {
             panic!("chunk index out of range");
         }
-        let slice = unsafe { std::slice::from_raw_parts(self.index.add(pos * 40 + 8), 32) };
-        slice.try_into().unwrap()
+        &self.index[pos].digest
     }
 
-    /// Compute checksum and data size
-    pub fn compute_csum(&self) -> ([u8; 32], u64) {
-        let mut csum = openssl::sha::Sha256::new();
-        let mut chunk_end = 0;
-        for pos in 0..self.index_entries {
-            chunk_end = self.chunk_end(pos);
-            let digest = self.chunk_digest(pos);
-            csum.update(&chunk_end.to_le_bytes());
-            csum.update(digest);
-        }
-        let csum = csum.finish();
-
-        (csum, chunk_end)
-    }
-
-    /*
-    pub fn dump_pxar(&self, mut writer: Box<dyn Write>) -> Result<(), Error> {
-
-        for pos in 0..self.index_entries {
-            let _end = self.chunk_end(pos);
-            let digest = self.chunk_digest(pos);
-            //println!("Dump {:08x}", end );
-            let chunk = self.store.read_chunk(digest)?;
-            // fimxe: handle encrypted chunks
-            let data = chunk.decode(None)?;
-            writer.write_all(&data)?;
-        }
-
-        Ok(())
-    }
-    */
-
+    // TODO: can we use std::slice::binary_search with Mmap now?
     fn binary_search(
         &self,
         start_idx: usize,
@@ -238,11 +187,11 @@ impl DynamicIndexReader {
 
 impl IndexFile for DynamicIndexReader {
     fn index_count(&self) -> usize {
-        self.index_entries
+        self.index.len()
     }
 
     fn index_digest(&self, pos: usize) -> Option<&[u8; 32]> {
-        if pos >= self.index_entries {
+        if pos >= self.index.len() {
             None
         } else {
             Some(unsafe { std::mem::transmute(self.chunk_digest(pos).as_ptr()) })
@@ -250,12 +199,77 @@ impl IndexFile for DynamicIndexReader {
     }
 
     fn index_bytes(&self) -> u64 {
-        if self.index_entries == 0 {
+        if self.index.is_empty() {
             0
         } else {
-            self.chunk_end((self.index_entries - 1) as usize)
+            self.chunk_end(self.index.len() - 1)
         }
     }
+
+    fn compute_csum(&self) -> ([u8; 32], u64) {
+        let mut csum = openssl::sha::Sha256::new();
+        let mut chunk_end = 0;
+        for pos in 0..self.index_count() {
+            let info = self.chunk_info(pos).unwrap();
+            chunk_end = info.range.end;
+            csum.update(&chunk_end.to_le_bytes());
+            csum.update(&info.digest);
+        }
+        let csum = csum.finish();
+        (csum, chunk_end)
+    }
+
+    #[allow(clippy::cast_ptr_alignment)]
+    fn chunk_info(&self, pos: usize) -> Option<ChunkReadInfo> {
+        if pos >= self.index.len() {
+            return None;
+        }
+        let start = if pos == 0 { 0 } else { self.index[pos - 1].end() };
+
+        let end = self.index[pos].end();
+
+        Some(ChunkReadInfo {
+            range: start..end,
+            digest: self.index[pos].digest.clone(),
+        })
+    }
+
+    fn chunk_from_offset(&self, offset: u64) -> Option<(usize, u64)> {
+        let end_idx = self.index.len() - 1;
+        let end = self.chunk_end(end_idx);
+        let found_idx = self.binary_search(0, 0, end_idx, end, offset);
+        let found_idx = match found_idx {
+            Ok(i) => i,
+            Err(_) => return None
+        };
+
+        let found_start = if found_idx == 0 {
+            0
+        } else {
+            self.chunk_end(found_idx - 1)
+        };
+
+        Some((found_idx, offset - found_start))
+    }
+}
+
+struct CachedChunk {
+    range: Range<u64>,
+    data: Vec<u8>,
+}
+
+impl CachedChunk {
+    /// Perform sanity checks on the range and data size:
+    pub fn new(range: Range<u64>, data: Vec<u8>) -> Result<Self, Error> {
+        if data.len() as u64 != range.end - range.start {
+            bail!(
+                "read chunk with wrong size ({} != {})",
+                data.len(),
+                range.end - range.start,
+            );
+        }
+        Ok(Self { range, data })
+    }
 }
 
 pub struct BufferedDynamicReader<S> {
@@ -266,7 +280,7 @@ pub struct BufferedDynamicReader<S> {
     buffered_chunk_idx: usize,
     buffered_chunk_start: u64,
     read_offset: u64,
-    lru_cache: crate::tools::lru_cache::LruCache<usize, (u64, u64, Vec<u8>)>,
+    lru_cache: crate::tools::lru_cache::LruCache<usize, CachedChunk>,
 }
 
 struct ChunkCacher<'a, S> {
@@ -274,16 +288,21 @@ struct ChunkCacher<'a, S> {
     index: &'a DynamicIndexReader,
 }
 
-impl<'a, S: ReadChunk> crate::tools::lru_cache::Cacher<usize, (u64, u64, Vec<u8>)> for ChunkCacher<'a, S> {
-    fn fetch(&mut self, index: usize) -> Result<Option<(u64, u64, Vec<u8>)>, anyhow::Error> {
-        let (start, end, digest) = self.index.chunk_info(index)?;
-        self.store.read_chunk(&digest).and_then(|data| Ok(Some((start, end, data))))
+impl<'a, S: ReadChunk> crate::tools::lru_cache::Cacher<usize, CachedChunk> for ChunkCacher<'a, S> {
+    fn fetch(&mut self, index: usize) -> Result<Option<CachedChunk>, Error> {
+        let info = match self.index.chunk_info(index) {
+            Some(info) => info,
+            None => bail!("chunk index out of range"),
+        };
+        let range = info.range;
+        let data = self.store.read_chunk(&info.digest)?;
+        CachedChunk::new(range, data).map(Some)
     }
 }
 
 impl<S: ReadChunk> BufferedDynamicReader<S> {
     pub fn new(index: DynamicIndexReader, store: S) -> Self {
-        let archive_size = index.chunk_end(index.index_entries - 1);
+        let archive_size = index.index_bytes();
         Self {
             store,
             index,
@@ -301,7 +320,8 @@ impl<S: ReadChunk> BufferedDynamicReader<S> {
     }
 
     fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> {
-        let (start, end, data) = self.lru_cache.access(
+        //let (start, end, data) = self.lru_cache.access(
+        let cached_chunk = self.lru_cache.access(
             idx,
             &mut ChunkCacher {
                 store: &mut self.store,
@@ -309,21 +329,13 @@ impl<S: ReadChunk> BufferedDynamicReader<S> {
             },
         )?.ok_or_else(|| format_err!("chunk not found by cacher"))?;
 
-        if (*end - *start) != data.len() as u64 {
-            bail!(
-                "read chunk with wrong size ({} != {}",
-                (*end - *start),
-                data.len()
-            );
-        }
-
         // fixme: avoid copy
         self.read_buffer.clear();
-        self.read_buffer.extend_from_slice(&data);
+        self.read_buffer.extend_from_slice(&cached_chunk.data);
 
         self.buffered_chunk_idx = idx;
 
-        self.buffered_chunk_start = *start;
+        self.buffered_chunk_start = cached_chunk.range.start;
         //println!("BUFFER {} {}",  self.buffered_chunk_start, end);
         Ok(())
     }
@@ -340,7 +352,7 @@ impl<S: ReadChunk> crate::tools::BufferedRead for BufferedDynamicReader<S> {
 
         // optimization for sequential read
         if buffer_len > 0
-            && ((self.buffered_chunk_idx + 1) < index.index_entries)
+            && ((self.buffered_chunk_idx + 1) < index.index.len())
             && (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
         {
             let next_idx = self.buffered_chunk_idx + 1;
@@ -356,7 +368,7 @@ impl<S: ReadChunk> crate::tools::BufferedRead for BufferedDynamicReader<S> {
             || (offset < self.buffered_chunk_start)
             || (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
         {
-            let end_idx = index.index_entries - 1;
+            let end_idx = index.index.len() - 1;
             let end = index.chunk_end(end_idx);
             let idx = index.binary_search(0, 0, end_idx, end, offset)?;
             self.buffer_chunk(idx)?;
@@ -383,9 +395,7 @@ impl<S: ReadChunk> std::io::Read for BufferedDynamicReader<S> {
             data.len()
         };
 
-        unsafe {
-            std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n);
-        }
+        buf[0..n].copy_from_slice(&data[0..n]);
 
         self.read_offset += n as u64;
 
@@ -417,6 +427,49 @@ impl<S: ReadChunk> std::io::Seek for BufferedDynamicReader<S> {
     }
 }
 
+/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
+/// async use!
+///
+/// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
+/// so that we can properly access it from multiple threads simultaneously while not issuing
+/// duplicate simultaneous reads over http.
+#[derive(Clone)]
+pub struct LocalDynamicReadAt<R: ReadChunk> {
+    inner: Arc<Mutex<BufferedDynamicReader<R>>>,
+}
+
+impl<R: ReadChunk> LocalDynamicReadAt<R> {
+    pub fn new(inner: BufferedDynamicReader<R>) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(inner)),
+        }
+    }
+}
+
+impl<R: ReadChunk> ReadAt for LocalDynamicReadAt<R> {
+    fn start_read_at<'a>(
+        self: Pin<&'a Self>,
+        _cx: &mut Context,
+        buf: &'a mut [u8],
+        offset: u64,
+    ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
+        use std::io::Read;
+        MaybeReady::Ready(tokio::task::block_in_place(move || {
+            let mut reader = self.inner.lock().unwrap();
+            reader.seek(SeekFrom::Start(offset))?;
+            Ok(reader.read(buf)?)
+        }))
+    }
+
+    fn poll_complete<'a>(
+        self: Pin<&'a Self>,
+        _op: ReadAtOperation<'a>,
+    ) -> MaybeReady<io::Result<usize>, ReadAtOperation<'a>> {
+        panic!("LocalDynamicReadAt::start_read_at returned Pending");
+    }
+}
+
+
 /// Create dynamic index files (`.dixd`)
 pub struct DynamicIndexWriter {
     store: Arc<ChunkStore>,
@@ -453,29 +506,16 @@ impl DynamicIndexWriter {
 
         let mut writer = BufWriter::with_capacity(1024 * 1024, file);
 
-        let header_size = std::mem::size_of::<DynamicIndexHeader>();
-
-        // todo: use static assertion when available in rust
-        if header_size != 4096 {
-            panic!("got unexpected header size");
-        }
-
-        let ctime = std::time::SystemTime::now()
-            .duration_since(std::time::SystemTime::UNIX_EPOCH)?
-            .as_secs();
+        let ctime = epoch_now_u64()?;
 
         let uuid = Uuid::generate();
 
-        let mut buffer = vec::zeroed(header_size);
-        let header = crate::tools::map_struct_mut::<DynamicIndexHeader>(&mut buffer)?;
-
+        let mut header = DynamicIndexHeader::zeroed();
         header.magic = super::DYNAMIC_SIZED_CHUNK_INDEX_1_0;
         header.ctime = u64::to_le(ctime);
         header.uuid = *uuid.as_bytes();
-
-        header.index_csum = [0u8; 32];
-
-        writer.write_all(&buffer)?;
+        // header.index_csum = [0u8; 32];
+        writer.write_all(header.as_bytes())?;
 
         let csum = Some(openssl::sha::Sha256::new());
 

src/backup/file_formats.rs

@@ -17,12 +17,6 @@ pub const ENCRYPTED_BLOB_MAGIC_1_0: [u8; 8] = [123, 103, 133, 190, 34, 45, 76, 2
 // openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted blob v1.0")[0..8]
 pub const ENCR_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [230, 89, 27, 191, 11, 191, 216, 11];
 
-//openssl::sha::sha256(b"Proxmox Backup authenticated blob v1.0")[0..8]
-pub const AUTHENTICATED_BLOB_MAGIC_1_0: [u8; 8] = [31, 135, 238, 226, 145, 206, 5, 2];
-
-//openssl::sha::sha256(b"Proxmox Backup zstd compressed authenticated blob v1.0")[0..8]
-pub const AUTH_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [126, 166, 15, 190, 145, 31, 169, 96];
-
 // openssl::sha::sha256(b"Proxmox Backup fixed sized chunk index v1.0")[0..8]
 pub const FIXED_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [47, 127, 65, 237, 145, 253, 15, 205];
 
@@ -50,19 +44,6 @@ pub struct DataBlobHeader {
     pub crc: [u8; 4],
 }
 
-/// Authenticated data blob binary storage format
-///
-/// The ``DataBlobHeader`` for authenticated blobs additionally contains
-/// a 16 byte HMAC tag, followed by the data:
-///
-/// (MAGIC || CRC32 || TAG || Data).
-#[derive(Endian)]
-#[repr(C,packed)]
-pub struct AuthenticatedDataBlobHeader {
-    pub head: DataBlobHeader,
-    pub tag: [u8; 32],
-}
-
 /// Encrypted data blob binary storage format
 ///
 /// The ``DataBlobHeader`` for encrypted blobs additionally contains
@@ -87,8 +68,6 @@ pub fn header_size(magic: &[u8; 8]) -> usize {
         &COMPRESSED_BLOB_MAGIC_1_0 => std::mem::size_of::<DataBlobHeader>(),
         &ENCRYPTED_BLOB_MAGIC_1_0 => std::mem::size_of::<EncryptedDataBlobHeader>(),
         &ENCR_COMPR_BLOB_MAGIC_1_0 => std::mem::size_of::<EncryptedDataBlobHeader>(),
-        &AUTHENTICATED_BLOB_MAGIC_1_0 => std::mem::size_of::<AuthenticatedDataBlobHeader>(),
-        &AUTH_COMPR_BLOB_MAGIC_1_0 => std::mem::size_of::<AuthenticatedDataBlobHeader>(),
         _ => panic!("unknown blob magic"),
     }
 }

src/backup/fixed_index.rs

@@ -1,11 +1,10 @@
 use anyhow::{bail, format_err, Error};
-use std::convert::TryInto;
 use std::io::{Seek, SeekFrom};
 
 use super::chunk_stat::*;
 use super::chunk_store::*;
-use super::IndexFile;
-use crate::tools;
+use super::{IndexFile, ChunkReadInfo};
+use crate::tools::{self, epoch_now_u64};
 
 use chrono::{Local, TimeZone};
 use std::fs::File;
@@ -14,7 +13,6 @@ use std::os::unix::io::AsRawFd;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use super::read_chunk::*;
 use super::ChunkInfo;
 
 use proxmox::tools::io::ReadExt;
@@ -147,66 +145,6 @@ impl FixedIndexReader {
         Ok(())
     }
 
-    pub fn chunk_info(&self, pos: usize) -> Result<(u64, u64, [u8; 32]), Error> {
-        if pos >= self.index_length {
-            bail!("chunk index out of range");
-        }
-        let start = (pos * self.chunk_size) as u64;
-        let mut end = start + self.chunk_size as u64;
-
-        if end > self.size {
-            end = self.size;
-        }
-
-        let mut digest = std::mem::MaybeUninit::<[u8; 32]>::uninit();
-        unsafe {
-            std::ptr::copy_nonoverlapping(
-                self.index.add(pos * 32),
-                (*digest.as_mut_ptr()).as_mut_ptr(),
-                32,
-            );
-        }
-
-        Ok((start, end, unsafe { digest.assume_init() }))
-    }
-
-    #[inline]
-    fn chunk_digest(&self, pos: usize) -> &[u8; 32] {
-        if pos >= self.index_length {
-            panic!("chunk index out of range");
-        }
-        let slice = unsafe { std::slice::from_raw_parts(self.index.add(pos * 32), 32) };
-        slice.try_into().unwrap()
-    }
-
-    #[inline]
-    fn chunk_end(&self, pos: usize) -> u64 {
-        if pos >= self.index_length {
-            panic!("chunk index out of range");
-        }
-
-        let end = ((pos + 1) * self.chunk_size) as u64;
-        if end > self.size {
-            self.size
-        } else {
-            end
-        }
-    }
-
-    /// Compute checksum and data size
-    pub fn compute_csum(&self) -> ([u8; 32], u64) {
-        let mut csum = openssl::sha::Sha256::new();
-        let mut chunk_end = 0;
-        for pos in 0..self.index_length {
-            chunk_end = ((pos + 1) * self.chunk_size) as u64;
-            let digest = self.chunk_digest(pos);
-            csum.update(digest);
-        }
-        let csum = csum.finish();
-
-        (csum, chunk_end)
-    }
-
     pub fn print_info(&self) {
         println!("Size: {}", self.size);
         println!("ChunkSize: {}", self.chunk_size);
@@ -234,6 +172,49 @@ impl IndexFile for FixedIndexReader {
     fn index_bytes(&self) -> u64 {
         self.size
     }
+
+    fn chunk_info(&self, pos: usize) -> Option<ChunkReadInfo> {
+        if pos >= self.index_length {
+            return None;
+        }
+
+        let start = (pos * self.chunk_size) as u64;
+        let mut end = start + self.chunk_size as u64;
+
+        if end > self.size {
+            end = self.size;
+        }
+
+        let digest = self.index_digest(pos).unwrap();
+        Some(ChunkReadInfo {
+            range: start..end,
+            digest: *digest,
+        })
+    }
+
+    fn compute_csum(&self) -> ([u8; 32], u64) {
+        let mut csum = openssl::sha::Sha256::new();
+        let mut chunk_end = 0;
+        for pos in 0..self.index_count() {
+            let info = self.chunk_info(pos).unwrap();
+            chunk_end = info.range.end;
+            csum.update(&info.digest);
+        }
+        let csum = csum.finish();
+
+        (csum, chunk_end)
+    }
+
+    fn chunk_from_offset(&self, offset: u64) -> Option<(usize, u64)> {
+        if offset >= self.size {
+            return None;
+        }
+
+        Some((
+            (offset / self.chunk_size as u64) as usize,
+            offset & (self.chunk_size - 1) as u64 // fast modulo, valid for 2^x chunk_size
+        ))
+    }
 }
 
 pub struct FixedIndexWriter {
@@ -290,9 +271,7 @@ impl FixedIndexWriter {
             panic!("got unexpected header size");
         }
 
-        let ctime = std::time::SystemTime::now()
-            .duration_since(std::time::SystemTime::UNIX_EPOCH)?
-            .as_secs();
+        let ctime = epoch_now_u64()?;
 
         let uuid = Uuid::generate();
 
@@ -469,145 +448,16 @@ impl FixedIndexWriter {
 
         Ok(())
     }
-}
 
-pub struct BufferedFixedReader<S> {
-    store: S,
-    index: FixedIndexReader,
-    archive_size: u64,
-    read_buffer: Vec<u8>,
-    buffered_chunk_idx: usize,
-    buffered_chunk_start: u64,
-    read_offset: u64,
-}
-
-impl<S: ReadChunk> BufferedFixedReader<S> {
-    pub fn new(index: FixedIndexReader, store: S) -> Self {
-        let archive_size = index.size;
-        Self {
-            store,
-            index,
-            archive_size,
-            read_buffer: Vec::with_capacity(1024 * 1024),
-            buffered_chunk_idx: 0,
-            buffered_chunk_start: 0,
-            read_offset: 0,
-        }
-    }
-
-    pub fn archive_size(&self) -> u64 {
-        self.archive_size
-    }
-
-    fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> {
-        let index = &self.index;
-        let (start, end, digest) = index.chunk_info(idx)?;
-
-        // fixme: avoid copy
-
-        let data = self.store.read_chunk(&digest)?;
-
-        if (end - start) != data.len() as u64 {
-            bail!(
-                "read chunk with wrong size ({} != {}",
-                (end - start),
-                data.len()
-            );
+    pub fn clone_data_from(&mut self, reader: &FixedIndexReader) -> Result<(), Error> {
+        if self.index_length != reader.index_count() {
+            bail!("clone_data_from failed - index sizes not equal");
         }
 
-        self.read_buffer.clear();
-        self.read_buffer.extend_from_slice(&data);
+        for i in 0..self.index_length {
+            self.add_digest(i, reader.index_digest(i).unwrap())?;
+        }
 
-        self.buffered_chunk_idx = idx;
-
-        self.buffered_chunk_start = start as u64;
-        //println!("BUFFER {} {}",  self.buffered_chunk_start, end);
         Ok(())
     }
 }
-
-impl<S: ReadChunk> crate::tools::BufferedRead for BufferedFixedReader<S> {
-    fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error> {
-        if offset == self.archive_size {
-            return Ok(&self.read_buffer[0..0]);
-        }
-
-        let buffer_len = self.read_buffer.len();
-        let index = &self.index;
-
-        // optimization for sequential read
-        if buffer_len > 0
-            && ((self.buffered_chunk_idx + 1) < index.index_length)
-            && (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
-        {
-            let next_idx = self.buffered_chunk_idx + 1;
-            let next_end = index.chunk_end(next_idx);
-            if offset < next_end {
-                self.buffer_chunk(next_idx)?;
-                let buffer_offset = (offset - self.buffered_chunk_start) as usize;
-                return Ok(&self.read_buffer[buffer_offset..]);
-            }
-        }
-
-        if (buffer_len == 0)
-            || (offset < self.buffered_chunk_start)
-            || (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
-        {
-            let idx = (offset / index.chunk_size as u64) as usize;
-            self.buffer_chunk(idx)?;
-        }
-
-        let buffer_offset = (offset - self.buffered_chunk_start) as usize;
-        Ok(&self.read_buffer[buffer_offset..])
-    }
-}
-
-impl<S: ReadChunk> std::io::Read for BufferedFixedReader<S> {
-    fn read(&mut self, buf: &mut [u8]) -> Result<usize, std::io::Error> {
-        use crate::tools::BufferedRead;
-        use std::io::{Error, ErrorKind};
-
-        let data = match self.buffered_read(self.read_offset) {
-            Ok(v) => v,
-            Err(err) => return Err(Error::new(ErrorKind::Other, err.to_string())),
-        };
-
-        let n = if data.len() > buf.len() {
-            buf.len()
-        } else {
-            data.len()
-        };
-
-        unsafe {
-            std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n);
-        }
-
-        self.read_offset += n as u64;
-
-        Ok(n)
-    }
-}
-
-impl<S: ReadChunk> Seek for BufferedFixedReader<S> {
-    fn seek(&mut self, pos: SeekFrom) -> Result<u64, std::io::Error> {
-        let new_offset = match pos {
-            SeekFrom::Start(start_offset) => start_offset as i64,
-            SeekFrom::End(end_offset) => (self.archive_size as i64) + end_offset,
-            SeekFrom::Current(offset) => (self.read_offset as i64) + offset,
-        };
-
-        use std::io::{Error, ErrorKind};
-        if (new_offset < 0) || (new_offset > (self.archive_size as i64)) {
-            return Err(Error::new(
-                ErrorKind::Other,
-                format!(
-                    "seek is out of range {} ([0..{}])",
-                    new_offset, self.archive_size
-                ),
-            ));
-        }
-        self.read_offset = new_offset as u64;
-
-        Ok(self.read_offset)
-    }
-}

src/backup/index.rs

@@ -1,10 +1,18 @@
 use std::collections::HashMap;
-use std::pin::Pin;
-use std::task::{Context, Poll};
+use std::ops::Range;
 
-use bytes::{Bytes, BytesMut};
-use anyhow::{format_err, Error};
-use futures::*;
+#[derive(Clone)]
+pub struct ChunkReadInfo {
+    pub range: Range<u64>,
+    pub digest: [u8; 32],
+}
+
+impl ChunkReadInfo {
+    #[inline]
+    pub fn size(&self) -> u64 {
+        self.range.end - self.range.start
+    }
+}
 
 /// Trait to get digest list from index files
 ///
@@ -13,6 +21,13 @@ pub trait IndexFile {
     fn index_count(&self) -> usize;
     fn index_digest(&self, pos: usize) -> Option<&[u8; 32]>;
     fn index_bytes(&self) -> u64;
+    fn chunk_info(&self, pos: usize) -> Option<ChunkReadInfo>;
+
+    /// Get the chunk index and the relative offset within it for a byte offset
+    fn chunk_from_offset(&self, offset: u64) -> Option<(usize, u64)>;
+
+    /// Compute index checksum and size
+    fn compute_csum(&self) -> ([u8; 32], u64);
 
     /// Returns most often used chunks
     fn find_most_used_chunks(&self, max: usize) -> HashMap<[u8; 32], usize> {
@@ -46,111 +61,3 @@ pub trait IndexFile {
         map
     }
 }
-
-/// Encode digest list from an `IndexFile` into a binary stream
-///
-/// The reader simply returns a birary stream of 32 byte digest values.
-pub struct DigestListEncoder {
-    index: Box<dyn IndexFile + Send + Sync>,
-    pos: usize,
-    count: usize,
-}
-
-impl DigestListEncoder {
-
-    pub fn new(index: Box<dyn IndexFile + Send + Sync>) -> Self {
-        let count = index.index_count();
-        Self { index, pos: 0, count }
-    }
-}
-
-impl std::io::Read for DigestListEncoder {
-    fn read(&mut self, buf: &mut [u8]) -> Result<usize, std::io::Error> {
-        if buf.len() < 32 {
-            panic!("read buffer too small");
-        }
-
-        if self.pos < self.count {
-            let mut written = 0;
-            loop {
-                let digest = self.index.index_digest(self.pos).unwrap();
-                buf[written..(written + 32)].copy_from_slice(digest);
-                self.pos += 1;
-                written += 32;
-                if self.pos >= self.count {
-                    break;
-                }
-                if (written + 32) >= buf.len() {
-                    break;
-                }
-            }
-            Ok(written)
-        } else {
-            Ok(0)
-        }
-    }
-}
-
-/// Decodes a Stream<Item=Bytes> into Stream<Item=<[u8;32]>
-///
-/// The reader simply returns a birary stream of 32 byte digest values.
-
-pub struct DigestListDecoder<S: Unpin> {
-    input: S,
-    buffer: BytesMut,
-}
-
-impl<S: Unpin> DigestListDecoder<S> {
-    pub fn new(input: S) -> Self {
-        Self { input, buffer: BytesMut::new() }
-    }
-}
-
-impl<S: Unpin> Unpin for DigestListDecoder<S> {}
-
-impl<S: Unpin, E> Stream for DigestListDecoder<S>
-where
-    S: Stream<Item=Result<Bytes, E>>,
-    E: Into<Error>,
-{
-    type Item = Result<[u8; 32], Error>;
-
-    fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
-        let this = self.get_mut();
-
-        loop {
-            if this.buffer.len() >= 32 {
-                let left = this.buffer.split_to(32);
-
-                let mut digest = std::mem::MaybeUninit::<[u8; 32]>::uninit();
-                unsafe {
-                    (*digest.as_mut_ptr()).copy_from_slice(&left[..]);
-                    return Poll::Ready(Some(Ok(digest.assume_init())));
-                }
-            }
-
-            match Pin::new(&mut this.input).poll_next(cx) {
-                Poll::Pending => {
-                    return Poll::Pending;
-                }
-                Poll::Ready(Some(Err(err))) => {
-                    return Poll::Ready(Some(Err(err.into())));
-                }
-                Poll::Ready(Some(Ok(data))) => {
-                    this.buffer.extend_from_slice(&data);
-                    // continue
-                }
-                Poll::Ready(None) => {
-                    let rest = this.buffer.len();
-                    if rest == 0 {
-                        return Poll::Ready(None);
-                    }
-                    return Poll::Ready(Some(Err(format_err!(
-                        "got small digest ({} != 32).",
-                        rest,
-                    ))));
-                }
-            }
-        }
-    }
-}

src/backup/key_derivation.rs

@@ -1,4 +1,4 @@
-use anyhow::{bail, format_err, Error};
+use anyhow::{bail, format_err, Context, Error};
 
 use serde::{Deserialize, Serialize};
 use chrono::{Local, TimeZone, DateTime};
@@ -146,12 +146,26 @@ pub fn encrypt_key_with_passphrase(
     })
 }
 
-pub fn load_and_decrypt_key(path: &std::path::Path, passphrase: &dyn Fn() -> Result<Vec<u8>, Error>) -> Result<([u8;32], DateTime<Local>), Error> {
+pub fn load_and_decrypt_key(
+    path: &std::path::Path,
+    passphrase: &dyn Fn() -> Result<Vec<u8>, Error>,
+) -> Result<([u8;32], DateTime<Local>), Error> {
+    do_load_and_decrypt_key(path, passphrase)
+        .with_context(|| format!("failed to load decryption key from {:?}", path))
+}
 
-    let raw = file_get_contents(&path)?;
-    let data = String::from_utf8(raw)?;
+fn do_load_and_decrypt_key(
+    path: &std::path::Path,
+    passphrase: &dyn Fn() -> Result<Vec<u8>, Error>,
+) -> Result<([u8;32], DateTime<Local>), Error> {
+    decrypt_key(&file_get_contents(&path)?, passphrase)
+}
 
-    let key_config: KeyConfig = serde_json::from_str(&data)?;
+pub fn decrypt_key(
+    mut keydata: &[u8],
+    passphrase: &dyn Fn() -> Result<Vec<u8>, Error>,
+) -> Result<([u8;32], DateTime<Local>), Error> {
+    let key_config: KeyConfig = serde_json::from_reader(&mut keydata)?;
 
     let raw_data = key_config.data;
     let created = key_config.created;

src/backup/manifest.rs

@@ -3,20 +3,76 @@ use std::convert::TryFrom;
 use std::path::Path;
 
 use serde_json::{json, Value};
+use ::serde::{Deserialize, Serialize};
 
-use crate::backup::BackupDir;
+use crate::backup::{BackupDir, CryptMode, CryptConfig};
 
 pub const MANIFEST_BLOB_NAME: &str = "index.json.blob";
+pub const CLIENT_LOG_BLOB_NAME: &str = "client.log.blob";
 
+mod hex_csum {
+    use serde::{self, Deserialize, Serializer, Deserializer};
+
+    pub fn serialize<S>(
+        csum: &[u8; 32],
+        serializer: S,
+    ) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let s = proxmox::tools::digest_to_hex(csum);
+        serializer.serialize_str(&s)
+    }
+
+    pub fn deserialize<'de, D>(
+        deserializer: D,
+    ) -> Result<[u8; 32], D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        proxmox::tools::hex_to_digest(&s).map_err(serde::de::Error::custom)
+    }
+}
+
+fn crypt_mode_none() -> CryptMode { CryptMode::None }
+fn empty_value() -> Value { json!({}) }
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all="kebab-case")]
 pub struct FileInfo {
     pub filename: String,
+    #[serde(default="crypt_mode_none")] // to be compatible with < 0.8.0 backups
+    pub crypt_mode: CryptMode,
     pub size: u64,
+    #[serde(with = "hex_csum")]
     pub csum: [u8; 32],
 }
 
+impl FileInfo {
+
+    /// Return expected CryptMode of referenced chunks
+    ///
+    /// Encrypted Indices should only reference encrypted chunks, while signed or plain indices
+    /// should only reference plain chunks.
+    pub fn chunk_crypt_mode (&self) -> CryptMode {
+        match self.crypt_mode {
+            CryptMode::Encrypt => CryptMode::Encrypt,
+            CryptMode::SignOnly | CryptMode::None => CryptMode::None,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all="kebab-case")]
 pub struct BackupManifest {
-    snapshot: BackupDir,
+    backup_type: String,
+    backup_id: String,
+    backup_time: i64,
     files: Vec<FileInfo>,
+    #[serde(default="empty_value")] // to be compatible with < 0.8.0 backups
+    pub unprotected: Value,
+    pub signature: Option<String>,
 }
 
 #[derive(PartialEq)]
@@ -44,12 +100,19 @@ pub fn archive_type<P: AsRef<Path>>(
 impl BackupManifest {
 
     pub fn new(snapshot: BackupDir) -> Self {
-        Self { files: Vec::new(), snapshot }
+        Self {
+            backup_type: snapshot.group().backup_type().into(),
+            backup_id: snapshot.group().backup_id().into(),
+            backup_time: snapshot.backup_time().timestamp(),
+            files: Vec::new(),
+            unprotected: json!({}),
+            signature: None,
+        }
     }
 
-    pub fn add_file(&mut self, filename: String, size: u64, csum: [u8; 32]) -> Result<(), Error> {
+    pub fn add_file(&mut self, filename: String, size: u64, csum: [u8; 32], crypt_mode: CryptMode) -> Result<(), Error> {
         let _archive_type = archive_type(&filename)?; // check type
-        self.files.push(FileInfo { filename, size, csum });
+        self.files.push(FileInfo { filename, size, csum, crypt_mode });
         Ok(())
     }
 
@@ -57,7 +120,7 @@ impl BackupManifest {
         &self.files[..]
     }
 
-    fn lookup_file_info(&self, name: &str) -> Result<&FileInfo, Error> {
+    pub fn lookup_file_info(&self, name: &str) -> Result<&FileInfo, Error> {
 
         let info = self.files.iter().find(|item| item.filename == name);
 
@@ -72,7 +135,7 @@ impl BackupManifest {
         let info = self.lookup_file_info(name)?;
 
         if size != info.size {
-            bail!("wrong size for file '{}' ({} != {}", name, info.size, size);
+            bail!("wrong size for file '{}' ({} != {})", name, info.size, size);
         }
 
         if csum != &info.csum {
@@ -82,66 +145,164 @@ impl BackupManifest {
         Ok(())
     }
 
-    pub fn into_json(self) -> Value {
-        json!({
-            "backup-type": self.snapshot.group().backup_type(),
-            "backup-id": self.snapshot.group().backup_id(),
-            "backup-time": self.snapshot.backup_time().timestamp(),
-            "files": self.files.iter()
-                .fold(Vec::new(), |mut acc, info| {
-                    acc.push(json!({
-                        "filename": info.filename,
-                        "size": info.size,
-                        "csum": proxmox::tools::digest_to_hex(&info.csum),
-                    }));
-                    acc
-                })
-        })
+    // Generate canonical json
+    fn to_canonical_json(value: &Value) -> Result<Vec<u8>, Error> {
+        let mut data = Vec::new();
+        Self::write_canonical_json(value, &mut data)?;
+        Ok(data)
     }
 
+    fn write_canonical_json(value: &Value, output: &mut Vec<u8>) -> Result<(), Error> {
+        match value {
+            Value::Null => bail!("got unexpected null value"),
+            Value::String(_) | Value::Number(_) | Value::Bool(_) => {
+                serde_json::to_writer(output, &value)?;
+            }
+            Value::Array(list) => {
+                output.push(b'[');
+                let mut iter = list.iter();
+                if let Some(item) = iter.next() {
+                    Self::write_canonical_json(item, output)?;
+                    for item in iter {
+                        output.push(b',');
+                        Self::write_canonical_json(item, output)?;
+                    }
+                }
+                output.push(b']');
+              }
+            Value::Object(map) => {
+                output.push(b'{');
+                let mut keys: Vec<&str> = map.keys().map(String::as_str).collect();
+                keys.sort();
+                let mut iter = keys.into_iter();
+                if let Some(key) = iter.next() {
+                    serde_json::to_writer(&mut *output, &key)?;
+                    output.push(b':');
+                    Self::write_canonical_json(&map[key], output)?;
+                    for key in iter {
+                        output.push(b',');
+                        serde_json::to_writer(&mut *output, &key)?;
+                        output.push(b':');
+                        Self::write_canonical_json(&map[key], output)?;
+                    }
+                }
+                output.push(b'}');
+            }
+        }
+        Ok(())
+    }
+
+    /// Compute manifest signature
+    ///
+    /// By generating a HMAC SHA256 over the canonical json
+    /// representation, The 'unpreotected' property is excluded.
+    pub fn signature(&self, crypt_config: &CryptConfig) -> Result<[u8; 32], Error> {
+        Self::json_signature(&serde_json::to_value(&self)?, crypt_config)
+    }
+
+    fn json_signature(data: &Value, crypt_config: &CryptConfig) -> Result<[u8; 32], Error> {
+
+        let mut signed_data = data.clone();
+
+        signed_data.as_object_mut().unwrap().remove("unprotected"); // exclude
+        signed_data.as_object_mut().unwrap().remove("signature"); // exclude
+
+        let canonical = Self::to_canonical_json(&signed_data)?;
+
+        let sig = crypt_config.compute_auth_tag(&canonical);
+
+        Ok(sig)
+    }
+
+    /// Converts the Manifest into json string, and add a signature if there is a crypt_config.
+    pub fn to_string(&self, crypt_config: Option<&CryptConfig>) -> Result<String, Error> {
+
+        let mut manifest = serde_json::to_value(&self)?;
+
+        if let Some(crypt_config) = crypt_config {
+            let sig = self.signature(crypt_config)?;
+            manifest["signature"] = proxmox::tools::digest_to_hex(&sig).into();
+        }
+
+        let manifest = serde_json::to_string_pretty(&manifest).unwrap().into();
+        Ok(manifest)
+    }
+
+    /// Try to read the manifest. This verifies the signature if there is a crypt_config.
+    pub fn from_data(data: &[u8], crypt_config: Option<&CryptConfig>) -> Result<BackupManifest, Error> {
+        let json: Value = serde_json::from_slice(data)?;
+        let signature = json["signature"].as_str().map(String::from);
+
+        if let Some(ref crypt_config) = crypt_config {
+            if let Some(signature) = signature {
+                let expected_signature = proxmox::tools::digest_to_hex(&Self::json_signature(&json, crypt_config)?);
+                if signature != expected_signature {
+                    bail!("wrong signature in manifest");
+                }
+            } else {
+                // not signed: warn/fail?
+            }
+        }
+
+        let manifest: BackupManifest = serde_json::from_value(json)?;
+        Ok(manifest)
+    }
 }
+
+
 impl TryFrom<super::DataBlob> for BackupManifest {
     type Error = Error;
 
     fn try_from(blob: super::DataBlob) -> Result<Self, Error> {
-        let data = blob.decode(None)
+        // no expected digest available
+        let data = blob.decode(None, None)
             .map_err(|err| format_err!("decode backup manifest blob failed - {}", err))?;
         let json: Value = serde_json::from_slice(&data[..])
             .map_err(|err| format_err!("unable to parse backup manifest json - {}", err))?;
-        BackupManifest::try_from(json)
+        let manifest: BackupManifest = serde_json::from_value(json)?;
+        Ok(manifest)
     }
 }
 
-impl TryFrom<Value> for BackupManifest {
-    type Error = Error;
 
-    fn try_from(data: Value) -> Result<Self, Error> {
+#[test]
+fn test_manifest_signature() -> Result<(), Error> {
 
-        use crate::tools::{required_string_property, required_integer_property, required_array_property};
+    use crate::backup::{KeyDerivationConfig};
 
-        proxmox::try_block!({
-            let backup_type = required_string_property(&data, "backup-type")?;
-            let backup_id = required_string_property(&data, "backup-id")?;
-            let backup_time = required_integer_property(&data, "backup-time")?;
+    let pw = b"test";
 
-            let snapshot = BackupDir::new(backup_type, backup_id, backup_time);
+    let kdf = KeyDerivationConfig::Scrypt {
+        n: 65536,
+        r: 8,
+        p: 1,
+        salt: Vec::new(),
+    };
 
-            let mut manifest = BackupManifest::new(snapshot);
+    let testkey = kdf.derive_key(pw)?;
 
-            for item in required_array_property(&data, "files")?.iter() {
-                let filename = required_string_property(item, "filename")?.to_owned();
-                let csum = required_string_property(item, "csum")?;
-                let csum = proxmox::tools::hex_to_digest(csum)?;
-                let size = required_integer_property(item, "size")? as u64;
-                manifest.add_file(filename, size, csum)?;
-            }
+    let crypt_config = CryptConfig::new(testkey)?;
 
-            if manifest.files().is_empty() {
-                bail!("manifest does not list any files.");
-            }
+    let snapshot: BackupDir = "host/elsa/2020-06-26T13:56:05Z".parse()?;
 
-            Ok(manifest)
-        }).map_err(|err: Error| format_err!("unable to parse backup manifest - {}", err))
+    let mut manifest = BackupManifest::new(snapshot);
 
-    }
+    manifest.add_file("test1.img.fidx".into(), 200, [1u8; 32], CryptMode::Encrypt)?;
+    manifest.add_file("abc.blob".into(), 200, [2u8; 32], CryptMode::None)?;
+
+    manifest.unprotected["note"] = "This is not protected by the signature.".into();
+
+    let text = manifest.to_string(Some(&crypt_config))?;
+
+    let manifest: Value = serde_json::from_str(&text)?;
+    let signature = manifest["signature"].as_str().unwrap().to_string();
+
+    assert_eq!(signature, "d7b446fb7db081662081d4b40fedd858a1d6307a5aff4ecff7d5bf4fd35679e9");
+
+    let manifest: BackupManifest = serde_json::from_value(manifest)?;
+    let expected_signature = proxmox::tools::digest_to_hex(&manifest.signature(&crypt_config)?);
+
+    assert_eq!(signature, expected_signature);
+
+    Ok(())
 }

src/backup/prune.rs

@@ -53,7 +53,7 @@ fn remove_incomplete_snapshots(
     let mut keep_unfinished = true;
     for info in list.iter() {
         // backup is considered unfinished if there is no manifest
-        if info.files.iter().any(|name| name == super::MANIFEST_BLOB_NAME) {
+        if info.is_finished() {
             // There is a new finished backup, so there is no need
             // to keep older unfinished backups.
             keep_unfinished = false;

src/backup/read_chunk.rs

@@ -1,53 +1,115 @@
-use anyhow::{Error};
+use std::future::Future;
+use std::pin::Pin;
 use std::sync::Arc;
 
-use super::datastore::*;
-use super::crypt_config::*;
-use super::data_blob::*;
+use anyhow::{bail, Error};
+
+use super::crypt_config::{CryptConfig, CryptMode};
+use super::data_blob::DataBlob;
+use super::datastore::DataStore;
 
 /// The ReadChunk trait allows reading backup data chunks (local or remote)
 pub trait ReadChunk {
     /// Returns the encoded chunk data
-    fn read_raw_chunk(&mut self, digest:&[u8; 32]) -> Result<DataBlob, Error>;
+    fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error>;
 
     /// Returns the decoded chunk data
-    fn read_chunk(&mut self, digest:&[u8; 32]) -> Result<Vec<u8>, Error>;
+    fn read_chunk(&self, digest: &[u8; 32]) -> Result<Vec<u8>, Error>;
 }
 
+#[derive(Clone)]
 pub struct LocalChunkReader {
     store: Arc<DataStore>,
     crypt_config: Option<Arc<CryptConfig>>,
+    crypt_mode: CryptMode,
 }
 
 impl LocalChunkReader {
+    pub fn new(store: Arc<DataStore>, crypt_config: Option<Arc<CryptConfig>>, crypt_mode: CryptMode) -> Self {
+        Self {
+            store,
+            crypt_config,
+            crypt_mode,
+        }
+    }
 
-    pub fn new(store: Arc<DataStore>, crypt_config: Option<Arc<CryptConfig>>) -> Self {
-        Self { store, crypt_config }
+    fn ensure_crypt_mode(&self, chunk_mode: CryptMode) -> Result<(), Error> {
+        match self.crypt_mode {
+            CryptMode::Encrypt => {
+                match chunk_mode {
+                    CryptMode::Encrypt => Ok(()),
+                    CryptMode::SignOnly | CryptMode::None => bail!("Index and chunk CryptMode don't match."),
+                }
+            },
+            CryptMode::SignOnly | CryptMode::None => {
+                match chunk_mode {
+                    CryptMode::Encrypt => bail!("Index and chunk CryptMode don't match."),
+                    CryptMode::SignOnly | CryptMode::None => Ok(()),
+                }
+            },
+        }
     }
 }
 
 impl ReadChunk for LocalChunkReader {
-
-    fn read_raw_chunk(&mut self, digest:&[u8; 32]) -> Result<DataBlob, Error> {
-
-        let digest_str = proxmox::tools::digest_to_hex(digest);
-        println!("READ CHUNK {}", digest_str);
-
-        let (path, _) = self.store.chunk_path(digest);
-        let raw_data = proxmox::tools::fs::file_get_contents(&path)?;
-        let chunk = DataBlob::from_raw(raw_data)?;
-        chunk.verify_crc()?;
-
+    fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> {
+        let chunk = self.store.load_chunk(digest)?;
+        self.ensure_crypt_mode(chunk.crypt_mode()?)?;
         Ok(chunk)
     }
 
-    fn read_chunk(&mut self, digest:&[u8; 32]) -> Result<Vec<u8>, Error> {
-        let chunk = self.read_raw_chunk(digest)?;
+    fn read_chunk(&self, digest: &[u8; 32]) -> Result<Vec<u8>, Error> {
+        let chunk = ReadChunk::read_raw_chunk(self, digest)?;
 
-        let raw_data =  chunk.decode(self.crypt_config.as_ref().map(Arc::as_ref))?;
-
-        // fixme: verify digest?
+        let raw_data = chunk.decode(self.crypt_config.as_ref().map(Arc::as_ref), Some(digest))?;
 
         Ok(raw_data)
     }
 }
+
+pub trait AsyncReadChunk: Send {
+    /// Returns the encoded chunk data
+    fn read_raw_chunk<'a>(
+        &'a self,
+        digest: &'a [u8; 32],
+    ) -> Pin<Box<dyn Future<Output = Result<DataBlob, Error>> + Send + 'a>>;
+
+    /// Returns the decoded chunk data
+    fn read_chunk<'a>(
+        &'a self,
+        digest: &'a [u8; 32],
+    ) -> Pin<Box<dyn Future<Output = Result<Vec<u8>, Error>> + Send + 'a>>;
+}
+
+impl AsyncReadChunk for LocalChunkReader {
+    fn read_raw_chunk<'a>(
+        &'a self,
+        digest: &'a [u8; 32],
+    ) -> Pin<Box<dyn Future<Output = Result<DataBlob, Error>> + Send + 'a>> {
+        Box::pin(async move{
+            let (path, _) = self.store.chunk_path(digest);
+
+            let raw_data = tokio::fs::read(&path).await?;
+
+            let chunk = DataBlob::load_from_reader(&mut &raw_data[..])?;
+            self.ensure_crypt_mode(chunk.crypt_mode()?)?;
+
+            Ok(chunk)
+        })
+    }
+
+    fn read_chunk<'a>(
+        &'a self,
+        digest: &'a [u8; 32],
+    ) -> Pin<Box<dyn Future<Output = Result<Vec<u8>, Error>> + Send + 'a>> {
+        Box::pin(async move {
+            let chunk = AsyncReadChunk::read_raw_chunk(self, digest).await?;
+
+            let raw_data = chunk.decode(self.crypt_config.as_ref().map(Arc::as_ref), Some(digest))?;
+
+            // fixme: verify digest?
+
+            Ok(raw_data)
+        })
+    }
+}

src/backup/verify.rs

@@ -0,0 +1,305 @@
+use std::collections::HashSet;
+
+use anyhow::{bail, format_err, Error};
+
+use crate::server::WorkerTask;
+use crate::api2::types::*;
+
+use super::{
+    DataStore, BackupGroup, BackupDir, BackupInfo, IndexFile,
+    CryptMode,
+    FileInfo, ArchiveType, archive_type,
+};
+
+fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
+
+    let blob = datastore.load_blob(backup_dir, &info.filename)?;
+
+    let raw_size = blob.raw_size();
+    if raw_size != info.size {
+        bail!("wrong size ({} != {})", info.size, raw_size);
+    }
+
+    let csum = openssl::sha::sha256(blob.raw_data());
+    if csum != info.csum {
+        bail!("wrong index checksum");
+    }
+
+    match blob.crypt_mode()? {
+        CryptMode::Encrypt => Ok(()),
+        CryptMode::None => {
+            // digest already verified above
+            blob.decode(None, None)?;
+            Ok(())
+        },
+        CryptMode::SignOnly => bail!("Invalid CryptMode for blob"),
+    }
+}
+
+fn verify_index_chunks(
+    datastore: &DataStore,
+    index: Box<dyn IndexFile>,
+    verified_chunks: &mut HashSet<[u8;32]>,
+    corrupt_chunks: &mut HashSet<[u8; 32]>,
+    crypt_mode: CryptMode,
+    worker: &WorkerTask,
+) -> Result<(), Error> {
+
+    let mut errors = 0;
+    for pos in 0..index.index_count() {
+
+        worker.fail_on_abort()?;
+
+        let info = index.chunk_info(pos).unwrap();
+
+        if verified_chunks.contains(&info.digest) {
+            continue; // already verified
+        }
+
+        if corrupt_chunks.contains(&info.digest) {
+            let digest_str = proxmox::tools::digest_to_hex(&info.digest);
+            worker.log(format!("chunk {} was marked as corrupt", digest_str));
+            errors += 1;
+            continue;
+        }
+
+        let chunk = match datastore.load_chunk(&info.digest) {
+            Err(err) => {
+                corrupt_chunks.insert(info.digest);
+                worker.log(format!("can't verify chunk, load failed - {}", err));
+                errors += 1;
+                continue;
+            },
+            Ok(chunk) => chunk,
+        };
+
+        let chunk_crypt_mode = match chunk.crypt_mode() {
+            Err(err) => {
+                corrupt_chunks.insert(info.digest);
+                worker.log(format!("can't verify chunk, unknown CryptMode - {}", err));
+                errors += 1;
+                continue;
+            },
+            Ok(mode) => mode,
+        };
+
+        if chunk_crypt_mode != crypt_mode {
+            worker.log(format!(
+                "chunk CryptMode {:?} does not match index CryptMode {:?}",
+                chunk_crypt_mode,
+                crypt_mode
+            ));
+            errors += 1;
+        }
+
+        let size = info.range.end - info.range.start;
+
+        if let Err(err) = chunk.verify_unencrypted(size as usize, &info.digest) {
+            corrupt_chunks.insert(info.digest);
+            worker.log(format!("{}", err));
+            errors += 1;
+        } else {
+            verified_chunks.insert(info.digest);
+        }
+    }
+
+    if errors > 0 {
+        bail!("chunks could not be verified");
+    }
+
+    Ok(())
+}
+
+fn verify_fixed_index(
+    datastore: &DataStore,
+    backup_dir: &BackupDir,
+    info: &FileInfo,
+    verified_chunks: &mut HashSet<[u8;32]>,
+    corrupt_chunks: &mut HashSet<[u8;32]>,
+    worker: &WorkerTask,
+) -> Result<(), Error> {
+
+    let mut path = backup_dir.relative_path();
+    path.push(&info.filename);
+
+    let index = datastore.open_fixed_reader(&path)?;
+
+    let (csum, size) = index.compute_csum();
+    if size != info.size {
+        bail!("wrong size ({} != {})", info.size, size);
+    }
+
+    if csum != info.csum {
+        bail!("wrong index checksum");
+    }
+
+    verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, info.chunk_crypt_mode(), worker)
+}
+
+fn verify_dynamic_index(
+    datastore: &DataStore,
+    backup_dir: &BackupDir,
+    info: &FileInfo,
+    verified_chunks: &mut HashSet<[u8;32]>,
+    corrupt_chunks: &mut HashSet<[u8;32]>,
+    worker: &WorkerTask,
+) -> Result<(), Error> {
+
+    let mut path = backup_dir.relative_path();
+    path.push(&info.filename);
+
+    let index = datastore.open_dynamic_reader(&path)?;
+
+    let (csum, size) = index.compute_csum();
+    if size != info.size {
+        bail!("wrong size ({} != {})", info.size, size);
+    }
+
+    if csum != info.csum {
+        bail!("wrong index checksum");
+    }
+
+    verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, info.chunk_crypt_mode(), worker)
+}
+
+/// Verify a single backup snapshot
+///
+/// This checks all archives inside a backup snapshot.
+/// Errors are logged to the worker log.
+///
+/// Returns
+/// - Ok(true) if verify is successful
+/// - Ok(false) if there were verification errors
+/// - Err(_) if task was aborted
+pub fn verify_backup_dir(
+    datastore: &DataStore,
+    backup_dir: &BackupDir,
+    verified_chunks: &mut HashSet<[u8;32]>,
+    corrupt_chunks: &mut HashSet<[u8;32]>,
+    worker: &WorkerTask
+) -> Result<bool, Error> {
+
+    let mut manifest = match datastore.load_manifest(&backup_dir) {
+        Ok((manifest, _)) => manifest,
+        Err(err) => {
+            worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err));
+            return Ok(false);
+        }
+    };
+
+    worker.log(format!("verify {}:{}", datastore.name(), backup_dir));
+
+    let mut error_count = 0;
+
+    let mut verify_result = "ok";
+    for info in manifest.files() {
+        let result = proxmox::try_block!({
+            worker.log(format!("  check {}", info.filename));
+            match archive_type(&info.filename)? {
+                ArchiveType::FixedIndex =>
+                    verify_fixed_index(
+                        &datastore,
+                        &backup_dir,
+                        info,
+                        verified_chunks,
+                        corrupt_chunks,
+                        worker
+                    ),
+                ArchiveType::DynamicIndex =>
+                    verify_dynamic_index(
+                        &datastore,
+                        &backup_dir,
+                        info,
+                        verified_chunks,
+                        corrupt_chunks,
+                        worker
+                    ),
+                ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info),
+            }
+        });
+
+        worker.fail_on_abort()?;
+
+        if let Err(err) = result {
+            worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err));
+            error_count += 1;
+            verify_result = "failed";
+        }
+
+    }
+
+    let verify_state = SnapshotVerifyState {
+        state: verify_result.to_string(),
+        upid: worker.upid().clone(),
+    };
+    manifest.unprotected["verify_state"] = serde_json::to_value(verify_state)?;
+    datastore.store_manifest(&backup_dir, serde_json::to_value(manifest)?)
+        .map_err(|err| format_err!("unable to store manifest blob - {}", err))?;
+
+
+    Ok(error_count == 0)
+}
+
+/// Verify all backups inside a backup group
+///
+/// Errors are logged to the worker log.
+///
+/// Returns
+/// - Ok(failed_dirs) where failed_dirs had verification errors
+/// - Err(_) if task was aborted
+pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<Vec<String>, Error> {
+
+    let mut errors = Vec::new();
+    let mut list = match group.list_backups(&datastore.base_path()) {
+        Ok(list) => list,
+        Err(err) => {
+            worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err));
+            return Ok(errors);
+        }
+    };
+
+    worker.log(format!("verify group {}:{}", datastore.name(), group));
+
+    let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB)
+    let mut corrupt_chunks = HashSet::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones
+
+    BackupInfo::sort_list(&mut list, false); // newest first
+    for info in list {
+        if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, &mut corrupt_chunks, worker)?{
+            errors.push(info.backup_dir.to_string());
+        }
+    }
+
+    Ok(errors)
+}
+
+/// Verify all backups inside a datastore
+///
+/// Errors are logged to the worker log.
+///
+/// Returns
+/// - Ok(failed_dirs) where failed_dirs had verification errors
+/// - Err(_) if task was aborted
+pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<Vec<String>, Error> {
+
+    let mut errors = Vec::new();
+
+    let mut list = match BackupGroup::list_groups(&datastore.base_path()) {
+        Ok(list) => list,
+        Err(err) => {
+            worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
+            return Ok(errors);
+        }
+    };
+
+    list.sort_unstable();
+
+    worker.log(format!("verify datastore {}", datastore.name()));
+
+    for group in list {
+        let mut group_errors = verify_backup_group(datastore, &group, worker)?;
+        errors.append(&mut group_errors);
+    }
+
+    Ok(errors)
+}

src/bin/proxmox-backup-api.rs

@@ -14,6 +14,8 @@ use proxmox_backup::config;
 use proxmox_backup::buildcfg;
 
 fn main() {
+    proxmox_backup::tools::setup_safe_path_env();
+
     if let Err(err) = proxmox_backup::tools::runtime::main(run()) {
         eprintln!("Error: {}", err);
         std::process::exit(-1);
@@ -35,6 +37,7 @@ async fn run() -> Result<(), Error> {
     config::update_self_signed_cert(false)?;
 
     proxmox_backup::rrd::create_rrdb_dir()?;
+    proxmox_backup::config::jobstate::create_jobstate_dir()?;
 
     if let Err(err) = generate_auth_key() {
         bail!("unable to generate auth key - {}", err);