tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

file-restore: add binary and basic commands

Browse Source 
For now it only supports 'list' and 'extract' commands for 'pxar.didx'
files. This should be the foundation for a general file-restore
interface that is shared with block-level snapshots.

This is packaged as a seperate .deb file, since for block level restore
it will need to depend on pve-qemu-kvm, which we want to seperate from
proxmox-backup-client.

[original code for proxmox-file-restore.rs]
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>

[code cleanups/clippy, use helpers::list_dir_content/ArchiveEntry, no
/block subdir for .fidx files, seperate binary and package]
Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
This commit is contained in:
Dominik Csapak 2021-03-31 12:21:48 +02:00 committed by Thomas Lamprecht
parent 42355b11a4
commit 76425d84b3
15 changed files with 457 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@ -61,7 +61,7 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
siphasher = "0.3"
syslog = "4.0"
tokio = { version = "1.0", features = [ "fs", "io-util", "macros", "net", "parking_lot", "process", "rt", "rt-multi-thread", "signal", "time" ] }
tokio = { version = "1.0", features = [ "fs", "io-util", "io-std", "macros", "net", "parking_lot", "process", "rt", "rt-multi-thread", "signal", "time" ] }
tokio-openssl = "0.6.1"
tokio-stream = "0.1.0"
tokio-util = { version = "0.6", features = [ "codec" ] }

9
Makefile
View File

@ -9,6 +9,7 @@ SUBDIRS := etc www docs
# Binaries usable by users
USR_BIN := \
proxmox-backup-client \
proxmox-file-restore \
pxar \
proxmox-tape \
pmtx \
@ -47,9 +48,12 @@ SERVER_DEB=${PACKAGE}-server_${DEB_VERSION}_${ARCH}.deb
SERVER_DBG_DEB=${PACKAGE}-server-dbgsym_${DEB_VERSION}_${ARCH}.deb
CLIENT_DEB=${PACKAGE}-client_${DEB_VERSION}_${ARCH}.deb
CLIENT_DBG_DEB=${PACKAGE}-client-dbgsym_${DEB_VERSION}_${ARCH}.deb
RESTORE_DEB=proxmox-file-restore_${DEB_VERSION}_${ARCH}.deb
RESTORE_DBG_DEB=proxmox-file-restore-dbgsym_${DEB_VERSION}_${ARCH}.deb
DOC_DEB=${PACKAGE}-docs_${DEB_VERSION}_all.deb
DEBS=${SERVER_DEB} ${SERVER_DBG_DEB} ${CLIENT_DEB} ${CLIENT_DBG_DEB}
DEBS=${SERVER_DEB} ${SERVER_DBG_DEB} ${CLIENT_DEB} ${CLIENT_DBG_DEB} \
${RESTORE_DEB} ${RESTORE_DBG_DEB}
DSC = rust-${PACKAGE}_${DEB_VERSION}.dsc
@ -152,8 +156,9 @@ install: $(COMPILED_BINS)
$(MAKE) -C docs install
.PHONY: upload
upload: ${SERVER_DEB} ${CLIENT_DEB} ${DOC_DEB}
upload: ${SERVER_DEB} ${CLIENT_DEB} ${RESTORE_DEB} ${DOC_DEB}
# check if working directory is clean
git diff --exit-code --stat && git diff --exit-code --stat --staged
tar cf - ${SERVER_DEB} ${SERVER_DBG_DEB} ${DOC_DEB} | ssh -X repoman@repo.proxmox.com upload --product pbs --dist buster
tar cf - ${CLIENT_DEB} ${CLIENT_DBG_DEB} | ssh -X repoman@repo.proxmox.com upload --product "pbs,pve,pmg" --dist buster
tar cf - ${RESTORE_DEB} ${RESTORE_DBG_DEB} | ssh -X repoman@repo.proxmox.com upload --product "pbs,pve,pmg" --dist buster

12
debian/control vendored
View File

@ -53,6 +53,7 @@ Build-Depends: debhelper (>= 11),
librust-syslog-4+default-dev,
librust-tokio-1+default-dev,
librust-tokio-1+fs-dev,
librust-tokio-1+io-std-dev,
librust-tokio-1+io-util-dev,
librust-tokio-1+macros-dev,
librust-tokio-1+net-dev,
@ -145,3 +146,14 @@ Depends: libjs-extjs,
Architecture: all
Description: Proxmox Backup Documentation
This package contains the Proxmox Backup Documentation files.
Package: proxmox-file-restore
Architecture: any
Depends: ${misc:Depends},
${shlibs:Depends},
proxmox-backup-restore-image,
Recommends: pve-qemu-kvm (>= 5.0.0-9),
Description: PBS single file restore for pxar and block device backups
This package contains the Proxmox Backup single file restore client for
restoring individual files and folders from both host/container and VM/block
device backups. It includes a block device restore driver using QEMU.

11
debian/control.in vendored
View File

@ -41,3 +41,14 @@ Depends: libjs-extjs,
Architecture: all
Description: Proxmox Backup Documentation
This package contains the Proxmox Backup Documentation files.
Package: proxmox-file-restore
Architecture: any
Depends: ${misc:Depends},
${shlibs:Depends},
proxmox-backup-restore-image,
Recommends: pve-qemu-kvm (>= 5.0.0-9),
Description: PBS single file restore for pxar and block device backups
This package contains the Proxmox Backup single file restore client for
restoring individual files and folders from both host/container and VM/block
device backups. It includes a block device restore driver using QEMU.

1
debian/proxmox-file-restore.bash-completion vendored Normal file
View File

@ -0,0 +1 @@
debian/proxmox-file-restore.bc proxmox-file-restore

8
debian/proxmox-file-restore.bc vendored Normal file
View File

@ -0,0 +1,8 @@
# proxmox-file-restore bash completion
# see http://tiswww.case.edu/php/chet/bash/FAQ
# and __ltrim_colon_completions() in /usr/share/bash-completion/bash_completion
# this modifies global var, but I found no better way
COMP_WORDBREAKS=${COMP_WORDBREAKS//:}
complete -C 'proxmox-file-restore bashcomplete' proxmox-file-restore

3
debian/proxmox-file-restore.install vendored Normal file
View File

@ -0,0 +1,3 @@
usr/bin/proxmox-file-restore
usr/share/man/man1/proxmox-file-restore.1
usr/share/zsh/vendor-completions/_proxmox-file-restore

7
debian/rules vendored
View File

@ -52,8 +52,11 @@ override_dh_dwz:
override_dh_strip:
dh_strip
for exe in $$(find debian/proxmox-backup-client/usr \
debian/proxmox-backup-server/usr -executable -type f); do \
for exe in $$(find \
debian/proxmox-backup-client/usr \
debian/proxmox-backup-server/usr \
debian/proxmox-file-restore/usr \
-executable -type f); do \
debian/scripts/elf-strip-unused-dependencies.sh "$$exe" || true; \
done

10
docs/Makefile
View File

@ -5,6 +5,7 @@ GENERATED_SYNOPSIS := \
proxmox-backup-client/synopsis.rst \
proxmox-backup-client/catalog-shell-synopsis.rst \
proxmox-backup-manager/synopsis.rst \
proxmox-file-restore/synopsis.rst \
pxar/synopsis.rst \
pmtx/synopsis.rst \
pmt/synopsis.rst \
@ -25,7 +26,8 @@ MAN1_PAGES := \
proxmox-tape.1 \
proxmox-backup-proxy.1 \
proxmox-backup-client.1 \
proxmox-backup-manager.1
proxmox-backup-manager.1 \
proxmox-file-restore.1
MAN5_PAGES := \
media-pool.cfg.5 \
@ -179,6 +181,12 @@ proxmox-backup-manager.1: proxmox-backup-manager/man1.rst proxmox-backup-manage
proxmox-backup-proxy.1: proxmox-backup-proxy/man1.rst proxmox-backup-proxy/description.rst
rst2man $< >$@
proxmox-file-restore/synopsis.rst: ${COMPILEDIR}/proxmox-file-restore
${COMPILEDIR}/proxmox-file-restore printdoc > proxmox-file-restore/synopsis.rst
proxmox-file-restore.1: proxmox-file-restore/man1.rst proxmox-file-restore/description.rst proxmox-file-restore/synopsis.rst
rst2man $< >$@
.PHONY: onlinehelpinfo
onlinehelpinfo:
@echo "Generating OnlineHelpInfo.js..."

5
docs/command-line-tools.rst
View File

@ -6,6 +6,11 @@ Command Line Tools
.. include:: proxmox-backup-client/description.rst
``proxmox-file-restore``
~~~~~~~~~~~~~~~~~~~~~~~~~
.. include:: proxmox-file-restore/description.rst
``proxmox-backup-manager``
~~~~~~~~~~~~~~~~~~~~~~~~~~

3
docs/proxmox-file-restore/description.rst Normal file
View File

@ -0,0 +1,3 @@
Command line tool for restoring files and directories from PBS archives. In contrast to
proxmox-backup-client, this supports both container/host and VM backups.

28
docs/proxmox-file-restore/man1.rst Normal file
View File

@ -0,0 +1,28 @@
==========================
proxmox-file-restore
==========================
.. include:: ../epilog.rst
-----------------------------------------------------------------------
Command line tool for restoring files and directories from PBS archives
-----------------------------------------------------------------------
:Author: |AUTHOR|
:Version: Version |VERSION|
:Manual section: 1
Synopsis
==========
.. include:: synopsis.rst
Description
============
.. include:: description.rst
.. include:: ../pbs-copyright.rst

2
src/api2.rs
View File

@ -12,7 +12,7 @@ pub mod version;
pub mod ping;
pub mod pull;
pub mod tape;
mod helpers;
pub mod helpers;
use proxmox::api::router::SubdirMap;
use proxmox::api::Router;

350
src/bin/proxmox-file-restore.rs Normal file
View File

@ -0,0 +1,350 @@
use std::ffi::OsStr;
use std::os::unix::ffi::OsStrExt;
use std::path::PathBuf;
use std::sync::Arc;
use anyhow::{bail, format_err, Error};
use serde_json::Value;
use proxmox::api::{
api,
cli::{run_cli_command, CliCommand, CliCommandMap, CliEnvironment},
};
use pxar::accessor::aio::Accessor;
use proxmox_backup::api2::{helpers, types::ArchiveEntry};
use proxmox_backup::backup::{
decrypt_key, BackupDir, BufferedDynamicReader, CatalogReader, CryptConfig, CryptMode,
DirEntryAttribute, IndexFile, LocalDynamicReadAt, CATALOG_NAME,
};
use proxmox_backup::client::{BackupReader, RemoteChunkReader};
use proxmox_backup::pxar::{create_zip, extract_sub_dir};
use proxmox_backup::tools;
// use "pub" so rust doesn't complain about "unused" functions in the module
pub mod proxmox_client_tools;
use proxmox_client_tools::{
complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value,
key_source::{
crypto_parameters, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
KEYFILE_SCHEMA,
},
REPO_URL_SCHEMA,
};
enum ExtractPath {
ListArchives,
Pxar(String, Vec<u8>),
}
fn parse_path(path: String, base64: bool) -> Result<ExtractPath, Error> {
let mut bytes = if base64 {
base64::decode(path)?
} else {
path.into_bytes()
};
if bytes == b"/" {
return Ok(ExtractPath::ListArchives);
}
while bytes.len() > 0 && bytes[0] == b'/' {
bytes.remove(0);
}
let (file, path) = {
let slash_pos = bytes.iter().position(|c| *c == b'/').unwrap_or(bytes.len());
let path = bytes.split_off(slash_pos);
let file = String::from_utf8(bytes)?;
(file, path)
};
if file.ends_with(".pxar.didx") {
Ok(ExtractPath::Pxar(file, path))
} else {
bail!("'{}' is not supported for file-restore", file);
}
}
#[api(
input: {
properties: {
repository: {
schema: REPO_URL_SCHEMA,
optional: true,
},
snapshot: {
type: String,
description: "Group/Snapshot path.",
},
"path": {
description: "Path to restore. Directories will be restored as .zip files.",
type: String,
},
"base64": {
type: Boolean,
description: "If set, 'path' will be interpreted as base64 encoded.",
optional: true,
default: false,
},
keyfile: {
schema: KEYFILE_SCHEMA,
optional: true,
},
"keyfd": {
schema: KEYFD_SCHEMA,
optional: true,
},
"crypt-mode": {
type: CryptMode,
optional: true,
},
}
}
)]
/// List a directory from a backup snapshot.
async fn list(
snapshot: String,
path: String,
base64: bool,
param: Value,
) -> Result<Vec<ArchiveEntry>, Error> {
let repo = extract_repository_from_value(&param)?;
let snapshot: BackupDir = snapshot.parse()?;
let path = parse_path(path, base64)?;
let crypto = crypto_parameters(&param)?;
let crypt_config = match crypto.enc_key {
None => None,
Some(ref key) => {
let (key, _, _) =
decrypt_key(&key.key, &get_encryption_key_password).map_err(|err| {
eprintln!("{}", format_key_source(&key.source, "encryption"));
err
})?;
Some(Arc::new(CryptConfig::new(key)?))
}
};
let client = connect(&repo)?;
let client = BackupReader::start(
client,
crypt_config.clone(),
repo.store(),
&snapshot.group().backup_type(),
&snapshot.group().backup_id(),
snapshot.backup_time(),
true,
)
.await?;
let (manifest, _) = client.download_manifest().await?;
manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
match path {
ExtractPath::ListArchives => {
let mut entries = vec![];
for file in manifest.files() {
match file.filename.rsplitn(2, '.').next().unwrap() {
"didx" => {}
"fidx" => {}
_ => continue, // ignore all non fidx/didx
}
let path = format!("/{}", file.filename);
let attr = DirEntryAttribute::Directory { start: 0 };
entries.push(ArchiveEntry::new(path.as_bytes(), &attr));
}
Ok(entries)
}
ExtractPath::Pxar(file, mut path) => {
let index = client
.download_dynamic_index(&manifest, CATALOG_NAME)
.await?;
let most_used = index.find_most_used_chunks(8);
let file_info = manifest.lookup_file_info(&CATALOG_NAME)?;
let chunk_reader = RemoteChunkReader::new(
client.clone(),
crypt_config,
file_info.chunk_crypt_mode(),
most_used,
);
let reader = BufferedDynamicReader::new(index, chunk_reader);
let mut catalog_reader = CatalogReader::new(reader);
let mut fullpath = file.into_bytes();
fullpath.append(&mut path);
helpers::list_dir_content(&mut catalog_reader, &fullpath)
}
}
}
#[api(
input: {
properties: {
repository: {
schema: REPO_URL_SCHEMA,
optional: true,
},
snapshot: {
type: String,
description: "Group/Snapshot path.",
},
"path": {
description: "Path to restore. Directories will be restored as .zip files if extracted to stdout.",
type: String,
},
"base64": {
type: Boolean,
description: "If set, 'path' will be interpreted as base64 encoded.",
optional: true,
default: false,
},
target: {
type: String,
optional: true,
description: "Target directory path. Use '-' to write to standard output.",
},
keyfile: {
schema: KEYFILE_SCHEMA,
optional: true,
},
"keyfd": {
schema: KEYFD_SCHEMA,
optional: true,
},
"crypt-mode": {
type: CryptMode,
optional: true,
},
verbose: {
type: Boolean,
description: "Print verbose information",
optional: true,
default: false,
}
}
}
)]
/// Restore files from a backup snapshot.
async fn extract(
snapshot: String,
path: String,
base64: bool,
target: Option<String>,
verbose: bool,
param: Value,
) -> Result<(), Error> {
let repo = extract_repository_from_value(&param)?;
let snapshot: BackupDir = snapshot.parse()?;
let orig_path = path;
let path = parse_path(orig_path.clone(), base64)?;
let target = match target {
Some(target) if target == "-" => None,
Some(target) => Some(PathBuf::from(target)),
None => Some(std::env::current_dir()?),
};
let crypto = crypto_parameters(&param)?;
let crypt_config = match crypto.enc_key {
None => None,
Some(ref key) => {
let (key, _, _) =
decrypt_key(&key.key, &get_encryption_key_password).map_err(|err| {
eprintln!("{}", format_key_source(&key.source, "encryption"));
err
})?;
Some(Arc::new(CryptConfig::new(key)?))
}
};
match path {
ExtractPath::Pxar(archive_name, path) => {
let client = connect(&repo)?;
let client = BackupReader::start(
client,
crypt_config.clone(),
repo.store(),
&snapshot.group().backup_type(),
&snapshot.group().backup_id(),
snapshot.backup_time(),
true,
)
.await?;
let (manifest, _) = client.download_manifest().await?;
let file_info = manifest.lookup_file_info(&archive_name)?;
let index = client
.download_dynamic_index(&manifest, &archive_name)
.await?;
let most_used = index.find_most_used_chunks(8);
let chunk_reader = RemoteChunkReader::new(
client.clone(),
crypt_config,
file_info.chunk_crypt_mode(),
most_used,
);
let reader = BufferedDynamicReader::new(index, chunk_reader);
let archive_size = reader.archive_size();
let reader = LocalDynamicReadAt::new(reader);
let decoder = Accessor::new(reader, archive_size).await?;
let root = decoder.open_root().await?;
let file = root
.lookup(OsStr::from_bytes(&path))
.await?
.ok_or(format_err!("error opening '{:?}'", path))?;
if let Some(target) = target {
extract_sub_dir(target, decoder, OsStr::from_bytes(&path), verbose).await?;
} else {
match file.kind() {
pxar::EntryKind::File { .. } => {
tokio::io::copy(&mut file.contents().await?, &mut tokio::io::stdout())
.await?;
}
_ => {
create_zip(
tokio::io::stdout(),
decoder,
OsStr::from_bytes(&path),
verbose,
)
.await?;
}
}
}
}
_ => {
bail!("cannot extract '{}'", orig_path);
}
}
Ok(())
}
fn main() {
let list_cmd_def = CliCommand::new(&API_METHOD_LIST)
.arg_param(&["snapshot", "path"])
.completion_cb("repository", complete_repository)
.completion_cb("snapshot", complete_group_or_snapshot);
let restore_cmd_def = CliCommand::new(&API_METHOD_EXTRACT)
.arg_param(&["snapshot", "path", "target"])
.completion_cb("repository", complete_repository)
.completion_cb("snapshot", complete_group_or_snapshot)
.completion_cb("target", tools::complete_file_name);
let cmd_def = CliCommandMap::new()
.insert("list", list_cmd_def)
.insert("extract", restore_cmd_def);
let rpcenv = CliEnvironment::new();
run_cli_command(
cmd_def,
rpcenv,
Some(|future| proxmox_backup::tools::runtime::main(future)),
);
}

13
zsh-completions/_proxmox-file-restore Normal file
View File

@ -0,0 +1,13 @@
#compdef _proxmox-backup-client() proxmox-backup-client
function _proxmox-backup-client() {
local cwords line point cmd curr prev
cworkds=${#words[@]}
line=$words
point=${#line}
cmd=${words[1]}
curr=${words[cwords]}
prev=${words[cwords-1]}
compadd -- $(COMP_CWORD="$cwords" COMP_LINE="$line" COMP_POINT="$point" \
proxmox-file-restore bashcomplete "$cmd" "$curr" "$prev")
}