switch to external pxar and fuse crates

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller
2020-03-23 15:03:18 +01:00
parent ab1092392f
commit c443f58b09
28 changed files with 4213 additions and 6191 deletions

View File

@ -1,66 +1,21 @@
extern crate proxmox_backup;
use std::collections::HashSet;
use std::ffi::OsStr;
use std::fs::OpenOptions;
use std::os::unix::fs::OpenOptionsExt;
use std::path::{Path, PathBuf};
use anyhow::{format_err, Error};
use futures::future::FutureExt;
use futures::select;
use tokio::signal::unix::{signal, SignalKind};
use pathpatterns::{MatchEntry, MatchType, PatternFlag};
use proxmox::{sortable, identity};
use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment};
use proxmox::api::schema::*;
use proxmox::api::cli::*;
use proxmox::api::api;
use proxmox_backup::tools;
use serde_json::{Value};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::fs::OpenOptions;
use std::ffi::OsStr;
use std::os::unix::fs::OpenOptionsExt;
use std::os::unix::io::AsRawFd;
use std::collections::HashSet;
use proxmox_backup::pxar;
fn dump_archive_from_reader<R: std::io::Read>(
reader: &mut R,
feature_flags: u64,
verbose: bool,
) -> Result<(), Error> {
let mut decoder = pxar::SequentialDecoder::new(reader, feature_flags);
let stdout = std::io::stdout();
let mut out = stdout.lock();
let mut path = PathBuf::new();
decoder.dump_entry(&mut path, verbose, &mut out)?;
Ok(())
}
fn dump_archive(
param: Value,
_info: &ApiMethod,
_rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
let archive = tools::required_string_param(&param, "archive")?;
let verbose = param["verbose"].as_bool().unwrap_or(false);
let feature_flags = pxar::flags::DEFAULT;
if archive == "-" {
let stdin = std::io::stdin();
let mut reader = stdin.lock();
dump_archive_from_reader(&mut reader, feature_flags, verbose)?;
} else {
if verbose { println!("PXAR dump: {}", archive); }
let file = std::fs::File::open(archive)?;
let mut reader = std::io::BufReader::new(file);
dump_archive_from_reader(&mut reader, feature_flags, verbose)?;
}
Ok(Value::Null)
}
use proxmox_backup::pxar::{flags, fuse, format_single_line_entry, ENCODER_MAX_ENTRIES};
fn extract_archive_from_reader<R: std::io::Read>(
reader: &mut R,
@ -68,124 +23,283 @@ fn extract_archive_from_reader<R: std::io::Read>(
feature_flags: u64,
allow_existing_dirs: bool,
verbose: bool,
pattern: Option<Vec<pxar::MatchPattern>>
match_list: &[MatchEntry],
) -> Result<(), Error> {
let mut decoder = pxar::SequentialDecoder::new(reader, feature_flags);
decoder.set_callback(move |path| {
if verbose {
println!("{:?}", path);
}
Ok(())
});
decoder.set_allow_existing_dirs(allow_existing_dirs);
let pattern = pattern.unwrap_or_else(Vec::new);
decoder.restore(Path::new(target), &pattern)?;
Ok(())
proxmox_backup::pxar::extract_archive(
pxar::decoder::Decoder::from_std(reader)?,
Path::new(target),
&match_list,
feature_flags,
allow_existing_dirs,
|path| {
if verbose {
println!("{:?}", path);
}
},
)
}
#[api(
input: {
properties: {
archive: {
description: "Archive name.",
},
pattern: {
description: "List of paths or pattern matching files to restore",
type: Array,
items: {
type: String,
description: "Path or pattern matching files to restore.",
},
optional: true,
},
target: {
description: "Target directory",
optional: true,
},
verbose: {
description: "Verbose output.",
optional: true,
default: false,
},
"no-xattrs": {
description: "Ignore extended file attributes.",
optional: true,
default: false,
},
"no-fcaps": {
description: "Ignore file capabilities.",
optional: true,
default: false,
},
"no-acls": {
description: "Ignore access control list entries.",
optional: true,
default: false,
},
"allow-existing-dirs": {
description: "Allows directories to already exist on restore.",
optional: true,
default: false,
},
"files-from": {
description: "File containing match pattern for files to restore.",
optional: true,
},
"no-device-nodes": {
description: "Ignore device nodes.",
optional: true,
default: false,
},
"no-fifos": {
description: "Ignore fifos.",
optional: true,
default: false,
},
"no-sockets": {
description: "Ignore sockets.",
optional: true,
default: false,
},
},
},
)]
/// Extract an archive.
fn extract_archive(
param: Value,
_info: &ApiMethod,
_rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
let archive = tools::required_string_param(&param, "archive")?;
let target = param["target"].as_str().unwrap_or(".");
let verbose = param["verbose"].as_bool().unwrap_or(false);
let no_xattrs = param["no-xattrs"].as_bool().unwrap_or(false);
let no_fcaps = param["no-fcaps"].as_bool().unwrap_or(false);
let no_acls = param["no-acls"].as_bool().unwrap_or(false);
let no_device_nodes = param["no-device-nodes"].as_bool().unwrap_or(false);
let no_fifos = param["no-fifos"].as_bool().unwrap_or(false);
let no_sockets = param["no-sockets"].as_bool().unwrap_or(false);
let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
let files_from = param["files-from"].as_str();
let empty = Vec::new();
let arg_pattern = param["pattern"].as_array().unwrap_or(&empty);
let mut feature_flags = pxar::flags::DEFAULT;
archive: String,
pattern: Option<Vec<String>>,
target: Option<String>,
verbose: bool,
no_xattrs: bool,
no_fcaps: bool,
no_acls: bool,
allow_existing_dirs: bool,
files_from: Option<String>,
no_device_nodes: bool,
no_fifos: bool,
no_sockets: bool,
) -> Result<(), Error> {
let mut feature_flags = flags::DEFAULT;
if no_xattrs {
feature_flags ^= pxar::flags::WITH_XATTRS;
feature_flags ^= flags::WITH_XATTRS;
}
if no_fcaps {
feature_flags ^= pxar::flags::WITH_FCAPS;
feature_flags ^= flags::WITH_FCAPS;
}
if no_acls {
feature_flags ^= pxar::flags::WITH_ACL;
feature_flags ^= flags::WITH_ACL;
}
if no_device_nodes {
feature_flags ^= pxar::flags::WITH_DEVICE_NODES;
feature_flags ^= flags::WITH_DEVICE_NODES;
}
if no_fifos {
feature_flags ^= pxar::flags::WITH_FIFOS;
feature_flags ^= flags::WITH_FIFOS;
}
if no_sockets {
feature_flags ^= pxar::flags::WITH_SOCKETS;
feature_flags ^= flags::WITH_SOCKETS;
}
let mut pattern_list = Vec::new();
if let Some(filename) = files_from {
let dir = nix::dir::Dir::open("./", nix::fcntl::OFlag::O_RDONLY, nix::sys::stat::Mode::empty())?;
if let Some((mut pattern, _, _)) = pxar::MatchPattern::from_file(dir.as_raw_fd(), filename)? {
pattern_list.append(&mut pattern);
let pattern = pattern.unwrap_or_else(Vec::new);
let target = target.as_ref().map_or_else(|| ".", String::as_str);
let mut match_list = Vec::new();
if let Some(filename) = &files_from {
for line in proxmox_backup::tools::file_get_non_comment_lines(filename)? {
let line = line
.map_err(|err| format_err!("error reading {}: {}", filename, err))?;
match_list.push(
MatchEntry::parse_pattern(line, PatternFlag::PATH_NAME, MatchType::Include)
.map_err(|err| format_err!("bad pattern in file '{}': {}", filename, err))?,
);
}
}
for s in arg_pattern {
let l = s.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
let p = pxar::MatchPattern::from_line(l.as_bytes())?
.ok_or_else(|| format_err!("Invalid match pattern in arguments"))?;
pattern_list.push(p);
for entry in pattern {
match_list.push(
MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Include)
.map_err(|err| format_err!("error in pattern: {}", err))?,
);
}
let pattern = if pattern_list.is_empty() {
None
} else {
Some(pattern_list)
};
if archive == "-" {
let stdin = std::io::stdin();
let mut reader = stdin.lock();
extract_archive_from_reader(&mut reader, target, feature_flags, allow_existing_dirs, verbose, pattern)?;
extract_archive_from_reader(
&mut reader,
&target,
feature_flags,
allow_existing_dirs,
verbose,
&match_list,
)?;
} else {
if verbose { println!("PXAR extract: {}", archive); }
if verbose {
println!("PXAR extract: {}", archive);
}
let file = std::fs::File::open(archive)?;
let mut reader = std::io::BufReader::new(file);
extract_archive_from_reader(&mut reader, target, feature_flags, allow_existing_dirs, verbose, pattern)?;
extract_archive_from_reader(
&mut reader,
&target,
feature_flags,
allow_existing_dirs,
verbose,
&match_list,
)?;
}
Ok(Value::Null)
Ok(())
}
#[api(
input: {
properties: {
archive: {
description: "Archive name.",
},
source: {
description: "Source directory.",
},
verbose: {
description: "Verbose output.",
optional: true,
default: false,
},
"no-xattrs": {
description: "Ignore extended file attributes.",
optional: true,
default: false,
},
"no-fcaps": {
description: "Ignore file capabilities.",
optional: true,
default: false,
},
"no-acls": {
description: "Ignore access control list entries.",
optional: true,
default: false,
},
"all-file-systems": {
description: "Include mounted sudirs.",
optional: true,
default: false,
},
"no-device-nodes": {
description: "Ignore device nodes.",
optional: true,
default: false,
},
"no-fifos": {
description: "Ignore fifos.",
optional: true,
default: false,
},
"no-sockets": {
description: "Ignore sockets.",
optional: true,
default: false,
},
exclude: {
description: "List of paths or pattern matching files to exclude.",
optional: true,
type: Array,
items: {
description: "Path or pattern matching files to restore",
type: String,
},
},
"entries-max": {
description: "Max number of entries loaded at once into memory",
optional: true,
default: ENCODER_MAX_ENTRIES as isize,
minimum: 0,
maximum: std::isize::MAX,
},
},
},
)]
/// Create a new .pxar archive.
fn create_archive(
param: Value,
_info: &ApiMethod,
_rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
archive: String,
source: String,
verbose: bool,
no_xattrs: bool,
no_fcaps: bool,
no_acls: bool,
all_file_systems: bool,
no_device_nodes: bool,
no_fifos: bool,
no_sockets: bool,
exclude: Option<Vec<String>>,
entries_max: isize,
) -> Result<(), Error> {
let exclude_list = {
let input = exclude.unwrap_or_else(Vec::new);
let mut exclude = Vec::with_capacity(input.len());
for entry in input {
exclude.push(
MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
.map_err(|err| format_err!("error in exclude pattern: {}", err))?,
);
}
exclude
};
let archive = tools::required_string_param(&param, "archive")?;
let source = tools::required_string_param(&param, "source")?;
let verbose = param["verbose"].as_bool().unwrap_or(false);
let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
let no_xattrs = param["no-xattrs"].as_bool().unwrap_or(false);
let no_fcaps = param["no-fcaps"].as_bool().unwrap_or(false);
let no_acls = param["no-acls"].as_bool().unwrap_or(false);
let no_device_nodes = param["no-device-nodes"].as_bool().unwrap_or(false);
let no_fifos = param["no-fifos"].as_bool().unwrap_or(false);
let no_sockets = param["no-sockets"].as_bool().unwrap_or(false);
let empty = Vec::new();
let exclude_pattern = param["exclude"].as_array().unwrap_or(&empty);
let entries_max = param["entries-max"].as_u64().unwrap_or(pxar::ENCODER_MAX_ENTRIES as u64);
let devices = if all_file_systems { None } else { Some(HashSet::new()) };
let device_set = if all_file_systems {
None
} else {
Some(HashSet::new())
};
let source = PathBuf::from(source);
let mut dir = nix::dir::Dir::open(
&source, nix::fcntl::OFlag::O_NOFOLLOW, nix::sys::stat::Mode::empty())?;
let dir = nix::dir::Dir::open(
&source,
nix::fcntl::OFlag::O_NOFOLLOW,
nix::sys::stat::Mode::empty(),
)?;
let file = OpenOptions::new()
.create_new(true)
@ -193,332 +307,150 @@ fn create_archive(
.mode(0o640)
.open(archive)?;
let mut writer = std::io::BufWriter::with_capacity(1024*1024, file);
let mut feature_flags = pxar::flags::DEFAULT;
let writer = std::io::BufWriter::with_capacity(1024 * 1024, file);
let mut feature_flags = flags::DEFAULT;
if no_xattrs {
feature_flags ^= pxar::flags::WITH_XATTRS;
feature_flags ^= flags::WITH_XATTRS;
}
if no_fcaps {
feature_flags ^= pxar::flags::WITH_FCAPS;
feature_flags ^= flags::WITH_FCAPS;
}
if no_acls {
feature_flags ^= pxar::flags::WITH_ACL;
feature_flags ^= flags::WITH_ACL;
}
if no_device_nodes {
feature_flags ^= pxar::flags::WITH_DEVICE_NODES;
feature_flags ^= flags::WITH_DEVICE_NODES;
}
if no_fifos {
feature_flags ^= pxar::flags::WITH_FIFOS;
feature_flags ^= flags::WITH_FIFOS;
}
if no_sockets {
feature_flags ^= pxar::flags::WITH_SOCKETS;
feature_flags ^= flags::WITH_SOCKETS;
}
let mut pattern_list = Vec::new();
for s in exclude_pattern {
let l = s.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
let p = pxar::MatchPattern::from_line(l.as_bytes())?
.ok_or_else(|| format_err!("Invalid match pattern in arguments"))?;
pattern_list.push(p);
}
let catalog = None::<&mut pxar::catalog::DummyCatalogWriter>;
pxar::Encoder::encode(
source,
&mut dir,
&mut writer,
catalog,
devices,
verbose,
false,
let writer = pxar::encoder::sync::StandardWriter::new(writer);
proxmox_backup::pxar::create_archive(
dir,
writer,
exclude_list,
feature_flags,
pattern_list,
device_set,
true,
|path| {
if verbose {
println!("{:?}", path);
}
Ok(())
},
entries_max as usize,
None,
)?;
writer.flush()?;
Ok(Value::Null)
Ok(())
}
#[api(
input: {
properties: {
archive: { description: "Archive name." },
mountpoint: { description: "Mountpoint for the file system." },
verbose: {
description: "Verbose output, running in the foreground (for debugging).",
optional: true,
default: false,
},
},
},
)]
/// Mount the archive to the provided mountpoint via FUSE.
fn mount_archive(
param: Value,
_info: &ApiMethod,
_rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
let archive = tools::required_string_param(&param, "archive")?;
let mountpoint = tools::required_string_param(&param, "mountpoint")?;
let verbose = param["verbose"].as_bool().unwrap_or(false);
let no_mt = param["no-mt"].as_bool().unwrap_or(false);
let archive = Path::new(archive);
let mountpoint = Path::new(mountpoint);
async fn mount_archive(
archive: String,
mountpoint: String,
verbose: bool,
) -> Result<(), Error> {
let archive = Path::new(&archive);
let mountpoint = Path::new(&mountpoint);
let options = OsStr::new("ro,default_permissions");
let mut session = pxar::fuse::Session::from_path(&archive, &options, verbose)
.map_err(|err| format_err!("pxar mount failed: {}", err))?;
// Mount the session and deamonize if verbose is not set
session.mount(&mountpoint, !verbose)?;
session.run_loop(!no_mt)?;
Ok(Value::Null)
let session = fuse::Session::mount_path(&archive, &options, verbose, mountpoint)
.await
.map_err(|err| format_err!("pxar mount failed: {}", err))?;
let mut interrupt = signal(SignalKind::interrupt())?;
select! {
res = session.fuse() => res?,
_ = interrupt.recv().fuse() => {
if verbose {
eprintln!("interrupted");
}
}
}
Ok(())
}
#[sortable]
const API_METHOD_CREATE_ARCHIVE: ApiMethod = ApiMethod::new(
&ApiHandler::Sync(&create_archive),
&ObjectSchema::new(
"Create new .pxar archive.",
&sorted!([
(
"archive",
false,
&StringSchema::new("Archive name").schema()
),
(
"source",
false,
&StringSchema::new("Source directory.").schema()
),
(
"verbose",
true,
&BooleanSchema::new("Verbose output.")
.default(false)
.schema()
),
(
"no-xattrs",
true,
&BooleanSchema::new("Ignore extended file attributes.")
.default(false)
.schema()
),
(
"no-fcaps",
true,
&BooleanSchema::new("Ignore file capabilities.")
.default(false)
.schema()
),
(
"no-acls",
true,
&BooleanSchema::new("Ignore access control list entries.")
.default(false)
.schema()
),
(
"all-file-systems",
true,
&BooleanSchema::new("Include mounted sudirs.")
.default(false)
.schema()
),
(
"no-device-nodes",
true,
&BooleanSchema::new("Ignore device nodes.")
.default(false)
.schema()
),
(
"no-fifos",
true,
&BooleanSchema::new("Ignore fifos.")
.default(false)
.schema()
),
(
"no-sockets",
true,
&BooleanSchema::new("Ignore sockets.")
.default(false)
.schema()
),
(
"exclude",
true,
&ArraySchema::new(
"List of paths or pattern matching files to exclude.",
&StringSchema::new("Path or pattern matching files to restore.").schema()
).schema()
),
(
"entries-max",
true,
&IntegerSchema::new("Max number of entries loaded at once into memory")
.default(pxar::ENCODER_MAX_ENTRIES as isize)
.minimum(0)
.maximum(std::isize::MAX)
.schema()
),
]),
)
);
#[api(
input: {
properties: {
archive: {
description: "Archive name.",
},
verbose: {
description: "Verbose output.",
optional: true,
default: false,
},
},
},
)]
/// List the contents of an archive.
fn dump_archive(archive: String, verbose: bool) -> Result<(), Error> {
for entry in pxar::decoder::Decoder::open(archive)? {
let entry = entry?;
#[sortable]
const API_METHOD_EXTRACT_ARCHIVE: ApiMethod = ApiMethod::new(
&ApiHandler::Sync(&extract_archive),
&ObjectSchema::new(
"Extract an archive.",
&sorted!([
(
"archive",
false,
&StringSchema::new("Archive name.").schema()
),
(
"pattern",
true,
&ArraySchema::new(
"List of paths or pattern matching files to restore",
&StringSchema::new("Path or pattern matching files to restore.").schema()
).schema()
),
(
"target",
true,
&StringSchema::new("Target directory.").schema()
),
(
"verbose",
true,
&BooleanSchema::new("Verbose output.")
.default(false)
.schema()
),
(
"no-xattrs",
true,
&BooleanSchema::new("Ignore extended file attributes.")
.default(false)
.schema()
),
(
"no-fcaps",
true,
&BooleanSchema::new("Ignore file capabilities.")
.default(false)
.schema()
),
(
"no-acls",
true,
&BooleanSchema::new("Ignore access control list entries.")
.default(false)
.schema()
),
(
"allow-existing-dirs",
true,
&BooleanSchema::new("Allows directories to already exist on restore.")
.default(false)
.schema()
),
(
"files-from",
true,
&StringSchema::new("Match pattern for files to restore.").schema()
),
(
"no-device-nodes",
true,
&BooleanSchema::new("Ignore device nodes.")
.default(false)
.schema()
),
(
"no-fifos",
true,
&BooleanSchema::new("Ignore fifos.")
.default(false)
.schema()
),
(
"no-sockets",
true,
&BooleanSchema::new("Ignore sockets.")
.default(false)
.schema()
),
]),
)
);
#[sortable]
const API_METHOD_MOUNT_ARCHIVE: ApiMethod = ApiMethod::new(
&ApiHandler::Sync(&mount_archive),
&ObjectSchema::new(
"Mount the archive as filesystem via FUSE.",
&sorted!([
(
"archive",
false,
&StringSchema::new("Archive name.").schema()
),
(
"mountpoint",
false,
&StringSchema::new("Mountpoint for the filesystem root.").schema()
),
(
"verbose",
true,
&BooleanSchema::new("Verbose output, keeps process running in foreground (for debugging).")
.default(false)
.schema()
),
(
"no-mt",
true,
&BooleanSchema::new("Run in single threaded mode (for debugging).")
.default(false)
.schema()
),
]),
)
);
#[sortable]
const API_METHOD_DUMP_ARCHIVE: ApiMethod = ApiMethod::new(
&ApiHandler::Sync(&dump_archive),
&ObjectSchema::new(
"List the contents of an archive.",
&sorted!([
( "archive", false, &StringSchema::new("Archive name.").schema()),
( "verbose", true, &BooleanSchema::new("Verbose output.")
.default(false)
.schema()
),
])
)
);
if verbose {
println!("{}", format_single_line_entry(&entry));
} else {
println!("{:?}", entry.path());
}
}
Ok(())
}
fn main() {
let cmd_def = CliCommandMap::new()
.insert("create", CliCommand::new(&API_METHOD_CREATE_ARCHIVE)
.arg_param(&["archive", "source"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("source", tools::complete_file_name)
.insert(
"create",
CliCommand::new(&API_METHOD_CREATE_ARCHIVE)
.arg_param(&["archive", "source"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("source", tools::complete_file_name),
)
.insert("extract", CliCommand::new(&API_METHOD_EXTRACT_ARCHIVE)
.arg_param(&["archive", "target"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("target", tools::complete_file_name)
.completion_cb("files-from", tools::complete_file_name)
)
.insert("mount", CliCommand::new(&API_METHOD_MOUNT_ARCHIVE)
.arg_param(&["archive", "mountpoint"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("mountpoint", tools::complete_file_name)
.insert(
"extract",
CliCommand::new(&API_METHOD_EXTRACT_ARCHIVE)
.arg_param(&["archive", "target"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("target", tools::complete_file_name)
.completion_cb("files-from", tools::complete_file_name),
)
.insert("list", CliCommand::new(&API_METHOD_DUMP_ARCHIVE)
.arg_param(&["archive"])
.completion_cb("archive", tools::complete_file_name)
.insert(
"mount",
CliCommand::new(&API_METHOD_MOUNT_ARCHIVE)
.arg_param(&["archive", "mountpoint"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("mountpoint", tools::complete_file_name),
)
.insert(
"list",
CliCommand::new(&API_METHOD_DUMP_ARCHIVE)
.arg_param(&["archive"])
.completion_cb("archive", tools::complete_file_name),
);
let rpcenv = CliEnvironment::new();
run_cli_command(cmd_def, rpcenv, None);
run_cli_command(cmd_def, rpcenv, Some(|future| {
proxmox_backup::tools::runtime::main(future)
}));
}