tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

src/catar/encoder.rs: auto-resize file_copy_buffer, limit number of dirs

Browse Source
This commit is contained in:
Dietmar Maurer 2018-12-30 14:09:59 +01:00
parent 4fa71e0573
commit 389e562524
1 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/catar/encoder.rs
View File

@ -23,7 +23,10 @@ use nix::sys::stat::FileStat;
use siphasher::sip::SipHasher24; use siphasher::sip::SipHasher24;
const FILE_COPY_BUFFER_SIZE: usize = 1024*1024; /// The format requires to build sorted directory lookup tables in
/// memory, so we restrict the number of allowed entries to limit
/// maximum memory usage.
pub const MAX_DIRECTORY_ENTRIES: usize = 256*1024;
pub struct CaTarEncoder<W: Write> { pub struct CaTarEncoder<W: Write> {
current_path: PathBuf, // used for error reporting current_path: PathBuf, // used for error reporting
@ -38,6 +41,8 @@ impl <W: Write> CaTarEncoder<W> {
pub fn encode(path: PathBuf, dir: &mut nix::dir::Dir, writer: W) -> Result<(), Error> { pub fn encode(path: PathBuf, dir: &mut nix::dir::Dir, writer: W) -> Result<(), Error> {
const FILE_COPY_BUFFER_SIZE: usize = 1024*1024;
let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE); let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
unsafe { file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE); } unsafe { file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE); }
@ -127,12 +132,13 @@ impl <W: Write> CaTarEncoder<W> {
self.write_header(CA_FORMAT_GOODBYE, goodbye_table_size as u64)?; self.write_header(CA_FORMAT_GOODBYE, goodbye_table_size as u64)?;
if goodbye_table_size > FILE_COPY_BUFFER_SIZE { if self.file_copy_buffer.capacity() < goodbye_table_size {
bail!("goodby table too large ({} > {})", goodbye_table_size, FILE_COPY_BUFFER_SIZE); let need = goodbye_table_size - self.file_copy_buffer.capacity();
self.file_copy_buffer.reserve(need);
unsafe { self.file_copy_buffer.set_len(self.file_copy_buffer.capacity()); }
} }
let buffer = &mut self.file_copy_buffer; let buffer = &mut self.file_copy_buffer;
let buffer_ptr = buffer.as_ptr();
copy_binary_search_tree(item_count, |s, d| { copy_binary_search_tree(item_count, |s, d| {
let item = &goodbye_items[s]; let item = &goodbye_items[s];
@ -176,7 +182,15 @@ impl <W: Write> CaTarEncoder<W> {
self.write_entry(&dir_stat)?; self.write_entry(&dir_stat)?;
let mut dir_count = 0;
for entry in dir.iter() { for entry in dir.iter() {
dir_count += 1;
if dir_count > MAX_DIRECTORY_ENTRIES {
bail!("too many directory items in {:?} (> {})",
self.current_path, MAX_DIRECTORY_ENTRIES);
}
let entry = match entry { let entry = match entry {
Ok(entry) => entry, Ok(entry) => entry,
Err(err) => bail!("readir {:?} failed - {}", self.current_path, err), Err(err) => bail!("readir {:?} failed - {}", self.current_path, err),