tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

switch to external pxar and fuse crates

Browse Source 
Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller
2020-03-23 15:03:18 +01:00
parent ab1092392f
commit c443f58b09
28 changed files with 4213 additions and 6191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
src/backup/catalog.rs
View File

@ -1,18 +1,18 @@
use anyhow::{bail, format_err, Error};
use std::fmt;
use std::ffi::{CStr, CString, OsStr};
use std::os::unix::ffi::OsStrExt;
use std::io::{Read, Write, Seek, SeekFrom};
use std::convert::TryFrom;
use std::ffi::{CStr, CString, OsStr};
use std::fmt;
use std::io::{Read, Write, Seek, SeekFrom};
use std::os::unix::ffi::OsStrExt;
use anyhow::{bail, format_err, Error};
use chrono::offset::{TimeZone, Local};
use proxmox::tools::io::ReadExt;
use pathpatterns::{MatchList, MatchType};
use proxmox::sys::error::io_err_other;
use proxmox::tools::io::ReadExt;
use crate::pxar::catalog::BackupCatalogWriter;
use crate::pxar::{MatchPattern, MatchPatternSlice, MatchType};
use crate::backup::file_formats::PROXMOX_CATALOG_FILE_MAGIC_1_0;
use crate::pxar::catalog::BackupCatalogWriter;
use crate::tools::runtime::block_on;
#[repr(u8)]
@ -63,7 +63,7 @@ pub struct DirEntry {
}
/// Used to specific additional attributes inside DirEntry
#[derive(Clone, PartialEq)]
#[derive(Clone, Debug, PartialEq)]
pub enum DirEntryAttribute {
Directory { start: u64 },
File { size: u64, mtime: u64 },
@ -106,6 +106,23 @@ impl DirEntry {
}
}
/// Get file mode bits for this entry to be used with the `MatchList` api.
pub fn get_file_mode(&self) -> Option<u32> {
Some(
match self.attr {
DirEntryAttribute::Directory { .. } => pxar::mode::IFDIR,
DirEntryAttribute::File { .. } => pxar::mode::IFREG,
DirEntryAttribute::Symlink => pxar::mode::IFLNK,
DirEntryAttribute::Hardlink => return None,
DirEntryAttribute::BlockDevice => pxar::mode::IFBLK,
DirEntryAttribute::CharDevice => pxar::mode::IFCHR,
DirEntryAttribute::Fifo => pxar::mode::IFIFO,
DirEntryAttribute::Socket => pxar::mode::IFSOCK,
}
as u32
)
}
/// Check if DirEntry is a directory
pub fn is_directory(&self) -> bool {
match self.attr {
@ -476,7 +493,7 @@ impl <R: Read + Seek> CatalogReader<R> {
&mut self,
parent: &DirEntry,
filename: &[u8],
) -> Result<DirEntry, Error> {
) -> Result<Option<DirEntry>, Error> {
let start = match parent.attr {
DirEntryAttribute::Directory { start } => start,
@ -496,10 +513,7 @@ impl <R: Read + Seek> CatalogReader<R> {
Ok(false) // stop parsing
})?;
match item {
None => bail!("no such file"),
Some(entry) => Ok(entry),
}
Ok(item)
}
/// Read the raw directory info block from current reader position.
@ -555,38 +569,30 @@ impl <R: Read + Seek> CatalogReader<R> {
/// provided callback on them.
pub fn find(
&mut self,
mut entry: &mut Vec<DirEntry>,
pattern: &[MatchPatternSlice],
callback: &Box<fn(&[DirEntry])>,
parent: &DirEntry,
file_path: &mut Vec<u8>,
match_list: &impl MatchList, //&[MatchEntry],
callback: &mut dyn FnMut(&[u8]) -> Result<(), Error>,
) -> Result<(), Error> {
let parent = entry.last().unwrap();
if !parent.is_directory() {
return Ok(())
}
let file_len = file_path.len();
for e in self.read_dir(parent)? {
match MatchPatternSlice::match_filename_include(
&CString::new(e.name.clone())?,
e.is_directory(),
pattern,
)? {
(MatchType::Positive, _) => {
entry.push(e);
callback(&entry);
let pattern = MatchPattern::from_line(b"**/*").unwrap().unwrap();
let child_pattern = vec![pattern.as_slice()];
self.find(&mut entry, &child_pattern, callback)?;
entry.pop();
}
(MatchType::PartialPositive, child_pattern)
| (MatchType::PartialNegative, child_pattern) => {
entry.push(e);
self.find(&mut entry, &child_pattern, callback)?;
entry.pop();
}
_ => {}
let is_dir = e.is_directory();
file_path.truncate(file_len);
if !e.name.starts_with(b"/") {
file_path.reserve(e.name.len() + 1);
file_path.push(b'/');
}
file_path.extend(&e.name);
match match_list.matches(&file_path, e.get_file_mode()) {
Some(MatchType::Exclude) => continue,
Some(MatchType::Include) => callback(&file_path)?,
None => (),
}
if is_dir {
self.find(&e, file_path, match_list, callback)?;
}
}
file_path.truncate(file_len);
Ok(())
}

1902
src/backup/catalog_shell.rs
View File

File diff suppressed because it is too large Load Diff

55
src/backup/dynamic_index.rs
View File

@ -1,6 +1,7 @@
use std::convert::TryInto;
use std::fs::File;
use std::io::{BufWriter, Seek, SeekFrom, Write};
use std::ops::Range;
use std::os::unix::io::AsRawFd;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@ -13,6 +14,7 @@ use proxmox::tools::vec;
use super::chunk_stat::ChunkStat;
use super::chunk_store::ChunkStore;
use super::index::ChunkReadInfo;
use super::read_chunk::ReadChunk;
use super::Chunker;
use super::IndexFile;
@ -136,7 +138,7 @@ impl DynamicIndexReader {
}
#[allow(clippy::cast_ptr_alignment)]
pub fn chunk_info(&self, pos: usize) -> Result<(u64, u64, [u8; 32]), Error> {
pub fn chunk_info(&self, pos: usize) -> Result<ChunkReadInfo, Error> {
if pos >= self.index_entries {
bail!("chunk index out of range");
}
@ -157,7 +159,10 @@ impl DynamicIndexReader {
);
}
Ok((start, end, unsafe { digest.assume_init() }))
Ok(ChunkReadInfo {
range: start..end,
digest: unsafe { digest.assume_init() },
})
}
#[inline]
@ -258,6 +263,25 @@ impl IndexFile for DynamicIndexReader {
}
}
struct CachedChunk {
range: Range<u64>,
data: Vec<u8>,
}
impl CachedChunk {
/// Perform sanity checks on the range and data size:
pub fn new(range: Range<u64>, data: Vec<u8>) -> Result<Self, Error> {
if data.len() as u64 != range.end - range.start {
bail!(
"read chunk with wrong size ({} != {})",
data.len(),
range.end - range.start,
);
}
Ok(Self { range, data })
}
}
pub struct BufferedDynamicReader<S> {
store: S,
index: DynamicIndexReader,
@ -266,7 +290,7 @@ pub struct BufferedDynamicReader<S> {
buffered_chunk_idx: usize,
buffered_chunk_start: u64,
read_offset: u64,
lru_cache: crate::tools::lru_cache::LruCache<usize, (u64, u64, Vec<u8>)>,
lru_cache: crate::tools::lru_cache::LruCache<usize, CachedChunk>,
}
struct ChunkCacher<'a, S> {
@ -274,10 +298,12 @@ struct ChunkCacher<'a, S> {
index: &'a DynamicIndexReader,
}
impl<'a, S: ReadChunk> crate::tools::lru_cache::Cacher<usize, (u64, u64, Vec<u8>)> for ChunkCacher<'a, S> {
fn fetch(&mut self, index: usize) -> Result<Option<(u64, u64, Vec<u8>)>, anyhow::Error> {
let (start, end, digest) = self.index.chunk_info(index)?;
self.store.read_chunk(&digest).and_then(|data| Ok(Some((start, end, data))))
impl<'a, S: ReadChunk> crate::tools::lru_cache::Cacher<usize, CachedChunk> for ChunkCacher<'a, S> {
fn fetch(&mut self, index: usize) -> Result<Option<CachedChunk>, Error> {
let info = self.index.chunk_info(index)?;
let range = info.range;
let data = self.store.read_chunk(&info.digest)?;
CachedChunk::new(range, data).map(Some)
}
}
@ -301,7 +327,8 @@ impl<S: ReadChunk> BufferedDynamicReader<S> {
}
fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> {
let (start, end, data) = self.lru_cache.access(
//let (start, end, data) = self.lru_cache.access(
let cached_chunk = self.lru_cache.access(
idx,
&mut ChunkCacher {
store: &mut self.store,
@ -309,21 +336,13 @@ impl<S: ReadChunk> BufferedDynamicReader<S> {
},
)?.ok_or_else(|| format_err!("chunk not found by cacher"))?;
if (*end - *start) != data.len() as u64 {
bail!(
"read chunk with wrong size ({} != {}",
(*end - *start),
data.len()
);
}
// fixme: avoid copy
self.read_buffer.clear();
self.read_buffer.extend_from_slice(&data);
self.read_buffer.extend_from_slice(&cached_chunk.data);
self.buffered_chunk_idx = idx;
self.buffered_chunk_start = *start;
self.buffered_chunk_start = cached_chunk.range.start;
//println!("BUFFER {} {}", self.buffered_chunk_start, end);
Ok(())
}

13
src/backup/index.rs
View File

@ -1,4 +1,5 @@
use std::collections::HashMap;
use std::ops::Range;
use std::pin::Pin;
use std::task::{Context, Poll};
@ -6,6 +7,18 @@ use bytes::{Bytes, BytesMut};
use anyhow::{format_err, Error};
use futures::*;
pub struct ChunkReadInfo {
pub range: Range<u64>,
pub digest: [u8; 32],
}
impl ChunkReadInfo {
#[inline]
pub fn size(&self) -> u64 {
self.range.end - self.range.start
}
}
/// Trait to get digest list from index files
///
/// To allow easy iteration over all used chunks.