tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

move client to pbs-client subcrate

Browse Source 
Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller
2021-07-19 10:50:18 +02:00
parent 72fbe9ffa5
commit 2b7f8dd5ea
74 changed files with 802 additions and 753 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
pbs-tools/Cargo.toml
View File

@ -9,6 +9,10 @@ description = "common tools used throughout pbs"
[dependencies]
anyhow = "1.0"
base64 = "0.12"
bytes = "1.0"
crc32fast = "1"
endian_trait = { version = "0.6", features = ["arrays"] }
flate2 = "1.0"
foreign-types = "0.3"
futures = "0.3"
lazy_static = "1.4"
@ -21,9 +25,10 @@ regex = "1.2"
serde = "1.0"
serde_json = "1.0"
# rt-multi-thread is required for block_in_place
tokio = { version = "1.6", features = [ "rt", "rt-multi-thread", "sync" ] }
tokio = { version = "1.6", features = [ "fs", "io-util", "rt", "rt-multi-thread", "sync" ] }
url = "2.1"
walkdir = "2"
proxmox = { version = "0.11.5", default-features = false, features = [] }
proxmox = { version = "0.11.5", default-features = false, features = [ "tokio" ] }
pbs-buildcfg = { path = "../pbs-buildcfg" }

334
pbs-tools/src/acl.rs Normal file
View File

@ -0,0 +1,334 @@
//! Implementation of the calls to handle POSIX access control lists
// see C header file <sys/acl.h> for reference
extern crate libc;
use std::ffi::CString;
use std::marker::PhantomData;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::io::RawFd;
use std::path::Path;
use std::ptr;
use libc::{c_char, c_int, c_uint, c_void};
use nix::errno::Errno;
use nix::NixPath;
// from: acl/include/acl.h
pub const ACL_UNDEFINED_ID: u32 = 0xffffffff;
// acl_perm_t values
pub type ACLPerm = c_uint;
pub const ACL_READ: ACLPerm = 0x04;
pub const ACL_WRITE: ACLPerm = 0x02;
pub const ACL_EXECUTE: ACLPerm = 0x01;
// acl_tag_t values
pub type ACLTag = c_int;
pub const ACL_UNDEFINED_TAG: ACLTag = 0x00;
pub const ACL_USER_OBJ: ACLTag = 0x01;
pub const ACL_USER: ACLTag = 0x02;
pub const ACL_GROUP_OBJ: ACLTag = 0x04;
pub const ACL_GROUP: ACLTag = 0x08;
pub const ACL_MASK: ACLTag = 0x10;
pub const ACL_OTHER: ACLTag = 0x20;
// acl_type_t values
pub type ACLType = c_uint;
pub const ACL_TYPE_ACCESS: ACLType = 0x8000;
pub const ACL_TYPE_DEFAULT: ACLType = 0x4000;
// acl entry constants
pub const ACL_FIRST_ENTRY: c_int = 0;
pub const ACL_NEXT_ENTRY: c_int = 1;
// acl to extended attribute names constants
// from: acl/include/acl_ea.h
pub const ACL_EA_ACCESS: &str = "system.posix_acl_access";
pub const ACL_EA_DEFAULT: &str = "system.posix_acl_default";
pub const ACL_EA_VERSION: u32 = 0x0002;
#[link(name = "acl")]
extern "C" {
fn acl_get_file(path: *const c_char, acl_type: ACLType) -> *mut c_void;
fn acl_set_file(path: *const c_char, acl_type: ACLType, acl: *mut c_void) -> c_int;
fn acl_get_fd(fd: RawFd) -> *mut c_void;
fn acl_get_entry(acl: *const c_void, entry_id: c_int, entry: *mut *mut c_void) -> c_int;
fn acl_create_entry(acl: *mut *mut c_void, entry: *mut *mut c_void) -> c_int;
fn acl_get_tag_type(entry: *mut c_void, tag_type: *mut ACLTag) -> c_int;
fn acl_set_tag_type(entry: *mut c_void, tag_type: ACLTag) -> c_int;
fn acl_get_permset(entry: *mut c_void, permset: *mut *mut c_void) -> c_int;
fn acl_clear_perms(permset: *mut c_void) -> c_int;
fn acl_get_perm(permset: *mut c_void, perm: ACLPerm) -> c_int;
fn acl_add_perm(permset: *mut c_void, perm: ACLPerm) -> c_int;
fn acl_get_qualifier(entry: *mut c_void) -> *mut c_void;
fn acl_set_qualifier(entry: *mut c_void, qualifier: *const c_void) -> c_int;
fn acl_init(count: c_int) -> *mut c_void;
fn acl_valid(ptr: *const c_void) -> c_int;
fn acl_free(ptr: *mut c_void) -> c_int;
}
#[derive(Debug)]
pub struct ACL {
ptr: *mut c_void,
}
impl Drop for ACL {
fn drop(&mut self) {
let ret = unsafe { acl_free(self.ptr) };
if ret != 0 {
panic!("invalid pointer encountered while dropping ACL - {}", Errno::last());
}
}
}
impl ACL {
pub fn init(count: usize) -> Result<ACL, nix::errno::Errno> {
let ptr = unsafe { acl_init(count as i32 as c_int) };
if ptr.is_null() {
return Err(Errno::last());
}
Ok(ACL { ptr })
}
pub fn get_file<P: AsRef<Path>>(path: P, acl_type: ACLType) -> Result<ACL, nix::errno::Errno> {
let path_cstr = CString::new(path.as_ref().as_os_str().as_bytes()).unwrap();
let ptr = unsafe { acl_get_file(path_cstr.as_ptr(), acl_type) };
if ptr.is_null() {
return Err(Errno::last());
}
Ok(ACL { ptr })
}
pub fn set_file<P: NixPath + ?Sized>(&self, path: &P, acl_type: ACLType) -> nix::Result<()> {
path.with_nix_path(|path| {
Errno::result(unsafe { acl_set_file(path.as_ptr(), acl_type, self.ptr) })
})?
.map(drop)
}
pub fn get_fd(fd: RawFd) -> Result<ACL, nix::errno::Errno> {
let ptr = unsafe { acl_get_fd(fd) };
if ptr.is_null() {
return Err(Errno::last());
}
Ok(ACL { ptr })
}
pub fn create_entry(&mut self) -> Result<ACLEntry, nix::errno::Errno> {
let mut ptr = ptr::null_mut() as *mut c_void;
let res = unsafe { acl_create_entry(&mut self.ptr, &mut ptr) };
if res < 0 {
return Err(Errno::last());
}
Ok(ACLEntry {
ptr,
_phantom: PhantomData,
})
}
pub fn is_valid(&self) -> bool {
let res = unsafe { acl_valid(self.ptr) };
if res == 0 {
return true;
}
false
}
pub fn entries(self) -> ACLEntriesIterator {
ACLEntriesIterator {
acl: self,
current: ACL_FIRST_ENTRY,
}
}
pub fn add_entry_full(&mut self, tag: ACLTag, qualifier: Option<u64>, permissions: u64)
-> Result<(), nix::errno::Errno>
{
let mut entry = self.create_entry()?;
entry.set_tag_type(tag)?;
if let Some(qualifier) = qualifier {
entry.set_qualifier(qualifier)?;
}
entry.set_permissions(permissions)?;
Ok(())
}
}
#[derive(Debug)]
pub struct ACLEntry<'a> {
ptr: *mut c_void,
_phantom: PhantomData<&'a mut ()>,
}
impl<'a> ACLEntry<'a> {
pub fn get_tag_type(&self) -> Result<ACLTag, nix::errno::Errno> {
let mut tag = ACL_UNDEFINED_TAG;
let res = unsafe { acl_get_tag_type(self.ptr, &mut tag as *mut ACLTag) };
if res < 0 {
return Err(Errno::last());
}
Ok(tag)
}
pub fn set_tag_type(&mut self, tag: ACLTag) -> Result<(), nix::errno::Errno> {
let res = unsafe { acl_set_tag_type(self.ptr, tag) };
if res < 0 {
return Err(Errno::last());
}
Ok(())
}
pub fn get_permissions(&self) -> Result<u64, nix::errno::Errno> {
let mut permissions = 0;
let mut permset = ptr::null_mut() as *mut c_void;
let mut res = unsafe { acl_get_permset(self.ptr, &mut permset) };
if res < 0 {
return Err(Errno::last());
}
for &perm in &[ACL_READ, ACL_WRITE, ACL_EXECUTE] {
res = unsafe { acl_get_perm(permset, perm) };
if res < 0 {
return Err(Errno::last());
}
if res == 1 {
permissions |= perm as u64;
}
}
Ok(permissions)
}
pub fn set_permissions(&mut self, permissions: u64) -> Result<u64, nix::errno::Errno> {
let mut permset = ptr::null_mut() as *mut c_void;
let mut res = unsafe { acl_get_permset(self.ptr, &mut permset) };
if res < 0 {
return Err(Errno::last());
}
res = unsafe { acl_clear_perms(permset) };
if res < 0 {
return Err(Errno::last());
}
for &perm in &[ACL_READ, ACL_WRITE, ACL_EXECUTE] {
if permissions & perm as u64 == perm as u64 {
res = unsafe { acl_add_perm(permset, perm) };
if res < 0 {
return Err(Errno::last());
}
}
}
Ok(permissions)
}
pub fn get_qualifier(&self) -> Result<u64, nix::errno::Errno> {
let qualifier = unsafe { acl_get_qualifier(self.ptr) };
if qualifier.is_null() {
return Err(Errno::last());
}
let result = unsafe { *(qualifier as *const u32) as u64 };
let ret = unsafe { acl_free(qualifier) };
if ret != 0 {
panic!("invalid pointer encountered while dropping ACL qualifier - {}", Errno::last());
}
Ok(result)
}
pub fn set_qualifier(&mut self, qualifier: u64) -> Result<(), nix::errno::Errno> {
let val = qualifier as u32;
let val_ptr: *const u32 = &val;
let res = unsafe { acl_set_qualifier(self.ptr, val_ptr as *const c_void) };
if res < 0 {
return Err(Errno::last());
}
Ok(())
}
}
#[derive(Debug)]
pub struct ACLEntriesIterator {
acl: ACL,
current: c_int,
}
impl<'a> Iterator for &'a mut ACLEntriesIterator {
type Item = ACLEntry<'a>;
fn next(&mut self) -> Option<Self::Item> {
let mut entry_ptr = ptr::null_mut();
let res = unsafe { acl_get_entry(self.acl.ptr, self.current, &mut entry_ptr) };
self.current = ACL_NEXT_ENTRY;
if res == 1 {
return Some(ACLEntry { ptr: entry_ptr, _phantom: PhantomData });
}
None
}
}
/// Helper to transform `PxarEntry`s user mode to acl permissions.
pub fn mode_user_to_acl_permissions(mode: u64) -> u64 {
(mode >> 6) & 7
}
/// Helper to transform `PxarEntry`s group mode to acl permissions.
pub fn mode_group_to_acl_permissions(mode: u64) -> u64 {
(mode >> 3) & 7
}
/// Helper to transform `PxarEntry`s other mode to acl permissions.
pub fn mode_other_to_acl_permissions(mode: u64) -> u64 {
mode & 7
}
/// Buffer to compose ACLs as extended attribute.
pub struct ACLXAttrBuffer {
buffer: Vec<u8>,
}
impl ACLXAttrBuffer {
/// Create a new buffer to write ACLs as extended attribute.
///
/// `version` defines the ACL_EA_VERSION found in acl/include/acl_ea.h
pub fn new(version: u32) -> Self {
let mut buffer = Vec::new();
buffer.extend_from_slice(&version.to_le_bytes());
Self { buffer }
}
/// Add ACL entry to buffer.
pub fn add_entry(&mut self, tag: ACLTag, qualifier: Option<u64>, permissions: u64) {
self.buffer.extend_from_slice(&(tag as u16).to_le_bytes());
self.buffer.extend_from_slice(&(permissions as u16).to_le_bytes());
match qualifier {
Some(qualifier) => self.buffer.extend_from_slice(&(qualifier as u32).to_le_bytes()),
None => self.buffer.extend_from_slice(&ACL_UNDEFINED_ID.to_le_bytes()),
}
}
/// Length of the buffer in bytes.
pub fn len(&self) -> usize {
self.buffer.len()
}
/// The buffer always contains at least the version, it is never empty
pub const fn is_empty(&self) -> bool { false }
/// Borrow raw buffer as mut slice.
pub fn as_mut_slice(&mut self) -> &mut [u8] {
self.buffer.as_mut_slice()
}
}

194
pbs-tools/src/compression.rs Normal file
View File

@ -0,0 +1,194 @@
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use anyhow::Error;
use bytes::Bytes;
use flate2::{Compress, Compression, FlushCompress};
use futures::ready;
use futures::stream::Stream;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use proxmox::io_format_err;
use proxmox::tools::byte_buffer::ByteBuffer;
const BUFFER_SIZE: usize = 8192;
pub enum Level {
Fastest,
Best,
Default,
Precise(u32),
}
#[derive(Eq, PartialEq)]
enum EncoderState {
Reading,
Writing,
Flushing,
Finished,
}
pub struct DeflateEncoder<T> {
inner: T,
compressor: Compress,
buffer: ByteBuffer,
input_buffer: Bytes,
state: EncoderState,
}
impl<T> DeflateEncoder<T> {
pub fn new(inner: T) -> Self {
Self::with_quality(inner, Level::Default)
}
pub fn with_quality(inner: T, level: Level) -> Self {
let level = match level {
Level::Fastest => Compression::fast(),
Level::Best => Compression::best(),
Level::Default => Compression::new(3),
Level::Precise(val) => Compression::new(val),
};
Self {
inner,
compressor: Compress::new(level, false),
buffer: ByteBuffer::with_capacity(BUFFER_SIZE),
input_buffer: Bytes::new(),
state: EncoderState::Reading,
}
}
pub fn total_in(&self) -> u64 {
self.compressor.total_in()
}
pub fn total_out(&self) -> u64 {
self.compressor.total_out()
}
pub fn into_inner(self) -> T {
self.inner
}
fn encode(
&mut self,
inbuf: &[u8],
flush: FlushCompress,
) -> Result<(usize, flate2::Status), io::Error> {
let old_in = self.compressor.total_in();
let old_out = self.compressor.total_out();
let res = self
.compressor
.compress(&inbuf[..], self.buffer.get_free_mut_slice(), flush)?;
let new_in = (self.compressor.total_in() - old_in) as usize;
let new_out = (self.compressor.total_out() - old_out) as usize;
self.buffer.add_size(new_out);
Ok((new_in, res))
}
}
impl DeflateEncoder<Vec<u8>> {
// assume small files
pub async fn compress_vec<R>(&mut self, reader: &mut R, size_hint: usize) -> Result<(), Error>
where
R: AsyncRead + Unpin,
{
let mut buffer = Vec::with_capacity(size_hint);
reader.read_to_end(&mut buffer).await?;
self.inner.reserve(size_hint); // should be enough since we want smalller files
self.compressor.compress_vec(&buffer[..], &mut self.inner, FlushCompress::Finish)?;
Ok(())
}
}
impl<T: AsyncWrite + Unpin> DeflateEncoder<T> {
pub async fn compress<R>(&mut self, reader: &mut R) -> Result<(), Error>
where
R: AsyncRead + Unpin,
{
let mut buffer = ByteBuffer::with_capacity(BUFFER_SIZE);
let mut eof = false;
loop {
if !eof && !buffer.is_full() {
let read = buffer.read_from_async(reader).await?;
if read == 0 {
eof = true;
}
}
let (read, _res) = self.encode(&buffer[..], FlushCompress::None)?;
buffer.consume(read);
self.inner.write_all(&self.buffer[..]).await?;
self.buffer.clear();
if buffer.is_empty() && eof {
break;
}
}
loop {
let (_read, res) = self.encode(&[][..], FlushCompress::Finish)?;
self.inner.write_all(&self.buffer[..]).await?;
self.buffer.clear();
if res == flate2::Status::StreamEnd {
break;
}
}
Ok(())
}
}
impl<T, O> Stream for DeflateEncoder<T>
where
T: Stream<Item = Result<O, io::Error>> + Unpin,
O: Into<Bytes>
{
type Item = Result<Bytes, io::Error>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
loop {
match this.state {
EncoderState::Reading => {
if let Some(res) = ready!(Pin::new(&mut this.inner).poll_next(cx)) {
let buf = res?;
this.input_buffer = buf.into();
this.state = EncoderState::Writing;
} else {
this.state = EncoderState::Flushing;
}
}
EncoderState::Writing => {
if this.input_buffer.is_empty() {
return Poll::Ready(Some(Err(io_format_err!("empty input during write"))));
}
let mut buf = this.input_buffer.split_off(0);
let (read, res) = this.encode(&buf[..], FlushCompress::None)?;
this.input_buffer = buf.split_off(read);
if this.input_buffer.is_empty() {
this.state = EncoderState::Reading;
}
if this.buffer.is_full() || res == flate2::Status::BufError {
let bytes = this.buffer.remove_data(this.buffer.len()).to_vec();
return Poll::Ready(Some(Ok(bytes.into())));
}
}
EncoderState::Flushing => {
let (_read, res) = this.encode(&[][..], FlushCompress::Finish)?;
if !this.buffer.is_empty() {
let bytes = this.buffer.remove_data(this.buffer.len()).to_vec();
return Poll::Ready(Some(Ok(bytes.into())));
}
if res == flate2::Status::StreamEnd {
this.state = EncoderState::Finished;
}
}
EncoderState::Finished => return Poll::Ready(None),
}
}
}
}

59
pbs-tools/src/fs.rs
View File

@ -1,13 +1,15 @@
//! File system helper utilities.
use std::borrow::{Borrow, BorrowMut};
use std::collections::HashMap;
use std::hash::BuildHasher;
use std::ops::{Deref, DerefMut};
use std::os::unix::io::{AsRawFd, RawFd};
use anyhow::{bail, format_err, Error};
use nix::dir;
use nix::dir::Dir;
use nix::fcntl::OFlag;
use nix::fcntl::{AtFlags, OFlag};
use nix::sys::stat::Mode;
use regex::Regex;
@ -344,3 +346,58 @@ fn do_lock_dir_noblock(
Ok(handle)
}
pub fn complete_file_name<S>(arg: &str, _param: &HashMap<String, String, S>) -> Vec<String>
where
S: BuildHasher,
{
let mut result = vec![];
let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg });
let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) {
Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR,
Err(_) => false,
};
if !is_dir {
if let Some(parent) = dirname.parent() {
dirname = parent.to_owned();
}
}
let mut dir =
match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) {
Ok(d) => d,
Err(_) => return result,
};
for item in dir.iter() {
if let Ok(entry) = item {
if let Ok(name) = entry.file_name().to_str() {
if name == "." || name == ".." {
continue;
}
let mut newpath = dirname.clone();
newpath.push(name);
if let Ok(stat) =
nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty())
{
if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR {
newpath.push("");
if let Some(newpath) = newpath.to_str() {
result.push(newpath.to_owned());
}
continue;
}
}
if let Some(newpath) = newpath.to_str() {
result.push(newpath.to_owned());
}
}
}
}
result
}

5
pbs-tools/src/lib.rs
View File

@ -1,11 +1,14 @@
pub mod acl;
pub mod auth;
pub mod borrow;
pub mod broadcast_future;
pub mod cert;
pub mod compression;
pub mod format;
pub mod fs;
pub mod json;
pub mod nom;
pub mod ops;
pub mod percent_encoding;
pub mod process_locker;
pub mod sha;
@ -13,6 +16,8 @@ pub mod str;
pub mod sync;
pub mod ticket;
pub mod tokio;
pub mod xattr;
pub mod zip;
mod command;
pub use command::{command_output, command_output_as_string, run_command};

12
pbs-tools/src/ops.rs Normal file
View File

@ -0,0 +1,12 @@
//! std::ops extensions
/// Modeled after the nightly `std::ops::ControlFlow`.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum ControlFlow<B, C = ()> {
Continue(C),
Break(B),
}
impl<B> ControlFlow<B> {
pub const CONTINUE: ControlFlow<B, ()> = ControlFlow::Continue(());
}

10
pbs-tools/src/str.rs
View File

@ -15,3 +15,13 @@ pub fn join<S: Borrow<str>>(data: &[S], sep: char) -> String {
list
}
pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] {
let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) {
Some(n) => &line[n..],
None => return &[],
};
match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) {
Some(n) => &line[..(line.len() - n)],
None => &[],
}
}

240
pbs-tools/src/xattr.rs Normal file
View File

@ -0,0 +1,240 @@
//! Wrapper functions for the libc xattr calls
use std::ffi::CStr;
use std::os::unix::io::RawFd;
use nix::errno::Errno;
use proxmox::c_str;
use proxmox::tools::vec;
/// `"security.capability"` as a CStr to avoid typos.
///
/// This cannot be `const` until `const_cstr_unchecked` is stable.
#[inline]
pub fn xattr_name_fcaps() -> &'static CStr {
c_str!("security.capability")
}
/// `"system.posix_acl_access"` as a CStr to avoid typos.
///
/// This cannot be `const` until `const_cstr_unchecked` is stable.
#[inline]
pub fn xattr_acl_access() -> &'static CStr {
c_str!("system.posix_acl_access")
}
/// `"system.posix_acl_default"` as a CStr to avoid typos.
///
/// This cannot be `const` until `const_cstr_unchecked` is stable.
#[inline]
pub fn xattr_acl_default() -> &'static CStr {
c_str!("system.posix_acl_default")
}
/// Result of `flistxattr`, allows iterating over the attributes as a list of `&CStr`s.
///
/// Listing xattrs produces a list separated by zeroes, inherently making them available as `&CStr`
/// already, so we make use of this fact and reflect this in the interface.
pub struct ListXAttr {
data: Vec<u8>,
}
impl ListXAttr {
fn new(data: Vec<u8>) -> Self {
Self { data }
}
}
impl<'a> IntoIterator for &'a ListXAttr {
type Item = &'a CStr;
type IntoIter = ListXAttrIter<'a>;
fn into_iter(self) -> Self::IntoIter {
ListXAttrIter {
data: &self.data,
at: 0,
}
}
}
/// Iterator over the extended attribute entries in a `ListXAttr`.
pub struct ListXAttrIter<'a> {
data: &'a [u8],
at: usize,
}
impl<'a> Iterator for ListXAttrIter<'a> {
type Item = &'a CStr;
fn next(&mut self) -> Option<&'a CStr> {
let data = &self.data[self.at..];
let next = data.iter().position(|b| *b == 0)? + 1;
self.at += next;
Some(unsafe { CStr::from_bytes_with_nul_unchecked(&data[..next]) })
}
}
/// Return a list of extended attributes accessible as an iterator over items of type `&CStr`.
pub fn flistxattr(fd: RawFd) -> Result<ListXAttr, nix::errno::Errno> {
// Initial buffer size for the attribute list, if content does not fit
// it gets dynamically increased until big enough.
let mut size = 256;
let mut buffer = vec::undefined(size);
let mut bytes = unsafe {
libc::flistxattr(fd, buffer.as_mut_ptr() as *mut libc::c_char, buffer.len())
};
while bytes < 0 {
let err = Errno::last();
match err {
Errno::ERANGE => {
// Buffer was not big enough to fit the list, retry with double the size
size = size.checked_mul(2).ok_or(Errno::ENOMEM)?;
},
_ => return Err(err),
}
// Retry to read the list with new buffer
buffer.resize(size, 0);
bytes = unsafe {
libc::flistxattr(fd, buffer.as_mut_ptr() as *mut libc::c_char, buffer.len())
};
}
buffer.truncate(bytes as usize);
Ok(ListXAttr::new(buffer))
}
/// Get an extended attribute by name.
///
/// Extended attributes may not contain zeroes, which we enforce in the API by using a `&CStr`
/// type.
pub fn fgetxattr(fd: RawFd, name: &CStr) -> Result<Vec<u8>, nix::errno::Errno> {
let mut size = 256;
let mut buffer = vec::undefined(size);
let mut bytes = unsafe {
libc::fgetxattr(fd, name.as_ptr(), buffer.as_mut_ptr() as *mut core::ffi::c_void, buffer.len())
};
while bytes < 0 {
let err = Errno::last();
match err {
Errno::ERANGE => {
// Buffer was not big enough to fit the value, retry with double the size
size = size.checked_mul(2).ok_or(Errno::ENOMEM)?;
},
_ => return Err(err),
}
buffer.resize(size, 0);
bytes = unsafe {
libc::fgetxattr(fd, name.as_ptr() as *const libc::c_char, buffer.as_mut_ptr() as *mut core::ffi::c_void, buffer.len())
};
}
buffer.resize(bytes as usize, 0);
Ok(buffer)
}
/// Set an extended attribute on a file descriptor.
pub fn fsetxattr(fd: RawFd, name: &CStr, data: &[u8]) -> Result<(), nix::errno::Errno> {
let flags = 0 as libc::c_int;
let result = unsafe {
libc::fsetxattr(fd, name.as_ptr(), data.as_ptr() as *const libc::c_void, data.len(), flags)
};
if result < 0 {
return Err(Errno::last());
}
Ok(())
}
pub fn fsetxattr_fcaps(fd: RawFd, fcaps: &[u8]) -> Result<(), nix::errno::Errno> {
// TODO casync checks and removes capabilities if they are set
fsetxattr(fd, xattr_name_fcaps(), fcaps)
}
pub fn is_security_capability(name: &CStr) -> bool {
name.to_bytes() == xattr_name_fcaps().to_bytes()
}
pub fn is_acl(name: &CStr) -> bool {
name.to_bytes() == xattr_acl_access().to_bytes()
|| name.to_bytes() == xattr_acl_default().to_bytes()
}
/// Check if the passed name buffer starts with a valid xattr namespace prefix
/// and is within the length limit of 255 bytes
pub fn is_valid_xattr_name(c_name: &CStr) -> bool {
let name = c_name.to_bytes();
if name.is_empty() || name.len() > 255 {
return false;
}
if name.starts_with(b"user.") || name.starts_with(b"trusted.") {
return true;
}
// samba saves windows ACLs there
if name == b"security.NTACL" {
return true;
}
is_security_capability(c_name)
}
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::CString;
use std::fs::OpenOptions;
use std::os::unix::io::AsRawFd;
use nix::errno::Errno;
use proxmox::c_str;
#[test]
fn test_fsetxattr_fgetxattr() {
let path = "./tests/xattrs.txt";
let file = OpenOptions::new()
.write(true)
.create(true)
.open(&path)
.unwrap();
let fd = file.as_raw_fd();
let mut name = b"user.".to_vec();
for _ in 0..260 {
name.push(b'a');
}
let invalid_name = CString::new(name).unwrap();
assert!(fsetxattr(fd, c_str!("user.attribute0"), b"value0").is_ok());
assert!(fsetxattr(fd, c_str!("user.empty"), b"").is_ok());
if nix::unistd::Uid::current() != nix::unistd::ROOT {
assert_eq!(fsetxattr(fd, c_str!("trusted.attribute0"), b"value0"), Err(Errno::EPERM));
}
assert_eq!(fsetxattr(fd, c_str!("garbage.attribute0"), b"value"), Err(Errno::EOPNOTSUPP));
assert_eq!(fsetxattr(fd, &invalid_name, b"err"), Err(Errno::ERANGE));
let v0 = fgetxattr(fd, c_str!("user.attribute0")).unwrap();
let v1 = fgetxattr(fd, c_str!("user.empty")).unwrap();
assert_eq!(v0, b"value0".as_ref());
assert_eq!(v1, b"".as_ref());
assert_eq!(fgetxattr(fd, c_str!("user.attribute1")), Err(Errno::ENODATA));
std::fs::remove_file(&path).unwrap();
}
#[test]
fn test_is_valid_xattr_name() {
let too_long = CString::new(vec![b'a'; 265]).unwrap();
assert!(!is_valid_xattr_name(&too_long));
assert!(!is_valid_xattr_name(c_str!("system.attr")));
assert!(is_valid_xattr_name(c_str!("user.attr")));
assert!(is_valid_xattr_name(c_str!("trusted.attr")));
assert!(is_valid_xattr_name(super::xattr_name_fcaps()));
}
}

671
pbs-tools/src/zip.rs Normal file
View File

@ -0,0 +1,671 @@
//! ZIP Helper
//!
//! Provides an interface to create a ZIP File from ZipEntries
//! for a more detailed description of the ZIP format, see:
//! https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
use std::convert::TryInto;
use std::ffi::OsString;
use std::io;
use std::mem::size_of;
use std::os::unix::ffi::OsStrExt;
use std::path::{Component, Path, PathBuf};
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::SystemTime;
use anyhow::{format_err, Error, Result};
use endian_trait::Endian;
use futures::ready;
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
use crc32fast::Hasher;
use proxmox::tools::time::gmtime;
use crate::compression::{DeflateEncoder, Level};
const LOCAL_FH_SIG: u32 = 0x04034B50;
const LOCAL_FF_SIG: u32 = 0x08074B50;
const CENTRAL_DIRECTORY_FH_SIG: u32 = 0x02014B50;
const END_OF_CENTRAL_DIR: u32 = 0x06054B50;
const VERSION_NEEDED: u16 = 0x002d;
const VERSION_MADE_BY: u16 = 0x032d;
const ZIP64_EOCD_RECORD: u32 = 0x06064B50;
const ZIP64_EOCD_LOCATOR: u32 = 0x07064B50;
// bits for time:
// 0-4: day of the month (1-31)
// 5-8: month: (1 = jan, etc.)
// 9-15: year offset from 1980
//
// bits for date:
// 0-4: second / 2
// 5-10: minute (0-59)
// 11-15: hour (0-23)
//
// see https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-filetimetodosdatetime
fn epoch_to_dos(epoch: i64) -> (u16, u16) {
let gmtime = match gmtime(epoch) {
Ok(gmtime) => gmtime,
Err(_) => return (0, 0),
};
let seconds = (gmtime.tm_sec / 2) & 0b11111;
let minutes = gmtime.tm_min & 0xb111111;
let hours = gmtime.tm_hour & 0b11111;
let time: u16 = ((hours << 11) | (minutes << 5) | (seconds)) as u16;
let date: u16 = if gmtime.tm_year > (2108 - 1900) || gmtime.tm_year < (1980 - 1900) {
0
} else {
let day = gmtime.tm_mday & 0b11111;
let month = (gmtime.tm_mon + 1) & 0b1111;
let year = (gmtime.tm_year + 1900 - 1980) & 0b1111111;
((year << 9) | (month << 5) | (day)) as u16
};
(date, time)
}
#[derive(Endian)]
#[repr(C, packed)]
struct Zip64Field {
field_type: u16,
field_size: u16,
uncompressed_size: u64,
compressed_size: u64,
}
#[derive(Endian)]
#[repr(C, packed)]
struct Zip64FieldWithOffset {
field_type: u16,
field_size: u16,
uncompressed_size: u64,
compressed_size: u64,
offset: u64,
start_disk: u32,
}
#[derive(Endian)]
#[repr(C, packed)]
struct LocalFileHeader {
signature: u32,
version_needed: u16,
flags: u16,
compression: u16,
time: u16,
date: u16,
crc32: u32,
compressed_size: u32,
uncompressed_size: u32,
filename_len: u16,
extra_field_len: u16,
}
#[derive(Endian)]
#[repr(C, packed)]
struct LocalFileFooter {
signature: u32,
crc32: u32,
compressed_size: u64,
uncompressed_size: u64,
}
#[derive(Endian)]
#[repr(C, packed)]
struct CentralDirectoryFileHeader {
signature: u32,
version_made_by: u16,
version_needed: u16,
flags: u16,
compression: u16,
time: u16,
date: u16,
crc32: u32,
compressed_size: u32,
uncompressed_size: u32,
filename_len: u16,
extra_field_len: u16,
comment_len: u16,
start_disk: u16,
internal_flags: u16,
external_flags: u32,
offset: u32,
}
#[derive(Endian)]
#[repr(C, packed)]
struct EndOfCentralDir {
signature: u32,
disk_number: u16,
start_disk: u16,
disk_record_count: u16,
total_record_count: u16,
directory_size: u32,
directory_offset: u32,
comment_len: u16,
}
#[derive(Endian)]
#[repr(C, packed)]
struct Zip64EOCDRecord {
signature: u32,
field_size: u64,
version_made_by: u16,
version_needed: u16,
disk_number: u32,
disk_number_central_dir: u32,
disk_record_count: u64,
total_record_count: u64,
directory_size: u64,
directory_offset: u64,
}
#[derive(Endian)]
#[repr(C, packed)]
struct Zip64EOCDLocator {
signature: u32,
disk_number: u32,
offset: u64,
disk_count: u32,
}
async fn write_struct<E, T>(output: &mut T, data: E) -> io::Result<()>
where
T: AsyncWrite + ?Sized + Unpin,
E: Endian,
{
let data = data.to_le();
let data = unsafe {
std::slice::from_raw_parts(
&data as *const E as *const u8,
core::mem::size_of_val(&data),
)
};
output.write_all(data).await
}
/// Represents an Entry in a ZIP File
///
/// used to add to a ZipEncoder
pub struct ZipEntry {
filename: OsString,
mtime: i64,
mode: u16,
crc32: u32,
uncompressed_size: u64,
compressed_size: u64,
offset: u64,
is_file: bool,
}
impl ZipEntry {
/// Creates a new ZipEntry
///
/// if is_file is false the path will contain an trailing separator,
/// so that the zip file understands that it is a directory
pub fn new<P: AsRef<Path>>(path: P, mtime: i64, mode: u16, is_file: bool) -> Self {
let mut relpath = PathBuf::new();
for comp in path.as_ref().components() {
if let Component::Normal(_) = comp {
relpath.push(comp);
}
}
if !is_file {
relpath.push(""); // adds trailing slash
}
Self {
filename: relpath.into(),
crc32: 0,
mtime,
mode,
uncompressed_size: 0,
compressed_size: 0,
offset: 0,
is_file,
}
}
async fn write_local_header<W>(&self, mut buf: &mut W) -> io::Result<usize>
where
W: AsyncWrite + Unpin + ?Sized,
{
let filename = self.filename.as_bytes();
let filename_len = filename.len();
let header_size = size_of::<LocalFileHeader>();
let zip_field_size = size_of::<Zip64Field>();
let size: usize = header_size + filename_len + zip_field_size;
let (date, time) = epoch_to_dos(self.mtime);
write_struct(
&mut buf,
LocalFileHeader {
signature: LOCAL_FH_SIG,
version_needed: 0x2d,
flags: 1 << 3,
compression: 0x8,
time,
date,
crc32: 0,
compressed_size: 0xFFFFFFFF,
uncompressed_size: 0xFFFFFFFF,
filename_len: filename_len as u16,
extra_field_len: zip_field_size as u16,
},
)
.await?;
buf.write_all(filename).await?;
write_struct(
&mut buf,
Zip64Field {
field_type: 0x0001,
field_size: 2 * 8,
uncompressed_size: 0,
compressed_size: 0,
},
)
.await?;
Ok(size)
}
async fn write_data_descriptor<W: AsyncWrite + Unpin + ?Sized>(
&self,
mut buf: &mut W,
) -> io::Result<usize> {
let size = size_of::<LocalFileFooter>();
write_struct(
&mut buf,
LocalFileFooter {
signature: LOCAL_FF_SIG,
crc32: self.crc32,
compressed_size: self.compressed_size,
uncompressed_size: self.uncompressed_size,
},
)
.await?;
Ok(size)
}
async fn write_central_directory_header<W: AsyncWrite + Unpin + ?Sized>(
&self,
mut buf: &mut W,
) -> io::Result<usize> {
let filename = self.filename.as_bytes();
let filename_len = filename.len();
let header_size = size_of::<CentralDirectoryFileHeader>();
let zip_field_size = size_of::<Zip64FieldWithOffset>();
let mut size: usize = header_size + filename_len;
let (date, time) = epoch_to_dos(self.mtime);
let (compressed_size, uncompressed_size, offset, need_zip64) = if self.compressed_size
>= (u32::MAX as u64)
|| self.uncompressed_size >= (u32::MAX as u64)
|| self.offset >= (u32::MAX as u64)
{
size += zip_field_size;
(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, true)
} else {
(
self.compressed_size as u32,
self.uncompressed_size as u32,
self.offset as u32,
false,
)
};
write_struct(
&mut buf,
CentralDirectoryFileHeader {
signature: CENTRAL_DIRECTORY_FH_SIG,
version_made_by: VERSION_MADE_BY,
version_needed: VERSION_NEEDED,
flags: 1 << 3,
compression: 0x8,
time,
date,
crc32: self.crc32,
compressed_size,
uncompressed_size,
filename_len: filename_len as u16,
extra_field_len: if need_zip64 { zip_field_size as u16 } else { 0 },
comment_len: 0,
start_disk: 0,
internal_flags: 0,
external_flags: (self.mode as u32) << 16 | (!self.is_file as u32) << 4,
offset,
},
)
.await?;
buf.write_all(filename).await?;
if need_zip64 {
write_struct(
&mut buf,
Zip64FieldWithOffset {
field_type: 1,
field_size: 3 * 8 + 4,
uncompressed_size: self.uncompressed_size,
compressed_size: self.compressed_size,
offset: self.offset,
start_disk: 0,
},
)
.await?;
}
Ok(size)
}
}
// wraps an asyncreader and calculates the hash
struct HashWrapper<R> {
inner: R,
hasher: Hasher,
}
impl<R> HashWrapper<R> {
fn new(inner: R) -> Self {
Self {
inner,
hasher: Hasher::new(),
}
}
// consumes self and returns the hash and the reader
fn finish(self) -> (u32, R) {
let crc32 = self.hasher.finalize();
(crc32, self.inner)
}
}
impl<R> AsyncRead for HashWrapper<R>
where
R: AsyncRead + Unpin,
{
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<Result<(), io::Error>> {
let this = self.get_mut();
let old_len = buf.filled().len();
ready!(Pin::new(&mut this.inner).poll_read(cx, buf))?;
let new_len = buf.filled().len();
if new_len > old_len {
this.hasher.update(&buf.filled()[old_len..new_len]);
}
Poll::Ready(Ok(()))
}
}
/// Wraps a writer that implements AsyncWrite for creating a ZIP archive
///
/// This will create a ZIP archive on the fly with files added with
/// 'add_entry'. To Finish the file, call 'finish'
/// Example:
/// ```no_run
/// use proxmox_backup::tools::zip::*;
/// use tokio::fs::File;
/// use anyhow::{Error, Result};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Error> {
/// let target = File::open("foo.zip").await?;
/// let mut source = File::open("foo.txt").await?;
///
/// let mut zip = ZipEncoder::new(target);
/// zip.add_entry(ZipEntry::new(
/// "foo.txt",
/// 0,
/// 0o100755,
/// true,
/// ), Some(source)).await?;
///
/// zip.finish().await?;
///
/// Ok(())
/// }
/// ```
pub struct ZipEncoder<W>
where
W: AsyncWrite + Unpin,
{
byte_count: usize,
files: Vec<ZipEntry>,
target: Option<W>,
}
impl<W: AsyncWrite + Unpin> ZipEncoder<W> {
pub fn new(target: W) -> Self {
Self {
byte_count: 0,
files: Vec::new(),
target: Some(target),
}
}
pub async fn add_entry<R: AsyncRead + Unpin>(
&mut self,
mut entry: ZipEntry,
content: Option<R>,
) -> Result<(), Error> {
let mut target = self
.target
.take()
.ok_or_else(|| format_err!("had no target during add entry"))?;
entry.offset = self.byte_count.try_into()?;
self.byte_count += entry.write_local_header(&mut target).await?;
if let Some(content) = content {
let mut reader = HashWrapper::new(content);
let mut enc = DeflateEncoder::with_quality(target, Level::Fastest);
enc.compress(&mut reader).await?;
let total_in = enc.total_in();
let total_out = enc.total_out();
target = enc.into_inner();
let (crc32, _reader) = reader.finish();
self.byte_count += total_out as usize;
entry.compressed_size = total_out;
entry.uncompressed_size = total_in;
entry.crc32 = crc32;
}
self.byte_count += entry.write_data_descriptor(&mut target).await?;
self.target = Some(target);
self.files.push(entry);
Ok(())
}
async fn write_eocd(
&mut self,
central_dir_size: usize,
central_dir_offset: usize,
) -> Result<(), Error> {
let entrycount = self.files.len();
let mut target = self
.target
.take()
.ok_or_else(|| format_err!("had no target during write_eocd"))?;
let mut count = entrycount as u16;
let mut directory_size = central_dir_size as u32;
let mut directory_offset = central_dir_offset as u32;
if central_dir_size > u32::MAX as usize
|| central_dir_offset > u32::MAX as usize
|| entrycount > u16::MAX as usize
{
count = 0xFFFF;
directory_size = 0xFFFFFFFF;
directory_offset = 0xFFFFFFFF;
write_struct(
&mut target,
Zip64EOCDRecord {
signature: ZIP64_EOCD_RECORD,
field_size: 44,
version_made_by: VERSION_MADE_BY,
version_needed: VERSION_NEEDED,
disk_number: 0,
disk_number_central_dir: 0,
disk_record_count: entrycount.try_into()?,
total_record_count: entrycount.try_into()?,
directory_size: central_dir_size.try_into()?,
directory_offset: central_dir_offset.try_into()?,
},
)
.await?;
let locator_offset = central_dir_offset + central_dir_size;
write_struct(
&mut target,
Zip64EOCDLocator {
signature: ZIP64_EOCD_LOCATOR,
disk_number: 0,
offset: locator_offset.try_into()?,
disk_count: 1,
},
)
.await?;
}
write_struct(
&mut target,
EndOfCentralDir {
signature: END_OF_CENTRAL_DIR,
disk_number: 0,
start_disk: 0,
disk_record_count: count,
total_record_count: count,
directory_size,
directory_offset,
comment_len: 0,
},
)
.await?;
self.target = Some(target);
Ok(())
}
pub async fn finish(&mut self) -> Result<(), Error> {
let mut target = self
.target
.take()
.ok_or_else(|| format_err!("had no target during finish"))?;
let central_dir_offset = self.byte_count;
let mut central_dir_size = 0;
for file in &self.files {
central_dir_size += file.write_central_directory_header(&mut target).await?;
}
self.target = Some(target);
self.write_eocd(central_dir_size, central_dir_offset)
.await?;
self.target
.take()
.ok_or_else(|| format_err!("had no target for flush"))?
.flush()
.await?;
Ok(())
}
}
/// Zip a local directory and write encoded data to target. "source" has to point to a valid
/// directory, it's name will be the root of the zip file - e.g.:
/// source:
/// /foo/bar
/// zip file:
/// /bar/file1
/// /bar/dir1
/// /bar/dir1/file2
/// ...
/// ...except if "source" is the root directory
pub async fn zip_directory<W>(target: W, source: &Path) -> Result<(), Error>
where
W: AsyncWrite + Unpin + Send,
{
use walkdir::WalkDir;
use std::os::unix::fs::MetadataExt;
let base_path = source.parent().unwrap_or_else(|| Path::new("/"));
let mut encoder = ZipEncoder::new(target);
for entry in WalkDir::new(&source).into_iter() {
match entry {
Ok(entry) => {
let entry_path = entry.path().to_owned();
let encoder = &mut encoder;
if let Err(err) = async move {
let entry_path_no_base = entry.path().strip_prefix(base_path)?;
let metadata = entry.metadata()?;
let mtime = match metadata.modified().unwrap_or_else(|_| SystemTime::now()).duration_since(SystemTime::UNIX_EPOCH) {
Ok(dur) => dur.as_secs() as i64,
Err(time_error) => -(time_error.duration().as_secs() as i64)
};
let mode = metadata.mode() as u16;
if entry.file_type().is_file() {
let file = tokio::fs::File::open(entry.path()).await?;
let ze = ZipEntry::new(
&entry_path_no_base,
mtime,
mode,
true,
);
encoder.add_entry(ze, Some(file)).await?;
} else if entry.file_type().is_dir() {
let ze = ZipEntry::new(
&entry_path_no_base,
mtime,
mode,
false,
);
let content: Option<tokio::fs::File> = None;
encoder.add_entry(ze, content).await?;
}
// ignore other file types
let ok: Result<(), Error> = Ok(());
ok
}
.await
{
eprintln!(
"zip: error encoding file or directory '{}': {}",
entry_path.display(),
err
);
}
}
Err(err) => {
eprintln!("zip: error reading directory entry: {}", err);
}
}
}
encoder.finish().await
}