tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

src/backup/chunk_store.rs - get_chunk_iterator: return percentage inside iterator item

Browse Source
This commit is contained in:
Dietmar Maurer 2019-07-04 09:26:44 +02:00
parent 0f0a35b390
commit a57360983b
2 changed files with 17 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/backup/chunk_store.rs
View File

@ -8,6 +8,7 @@ use serde::Serialize;
use crate::tools; use crate::tools;
use super::DataChunk; use super::DataChunk;
use crate::server::WorkerTask;
#[derive(Clone, Serialize)] #[derive(Clone, Serialize)]
pub struct GarbageCollectionStatus { pub struct GarbageCollectionStatus {
@ -184,9 +185,8 @@ impl ChunkStore {
pub fn get_chunk_iterator( pub fn get_chunk_iterator(
&self, &self,
print_percentage: bool,
) -> Result< ) -> Result<
impl Iterator<Item = Result<tools::fs::ReadDirEntry, Error>> + std::iter::FusedIterator, impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize)> + std::iter::FusedIterator,
Error Error
> { > {
use nix::dir::Dir; use nix::dir::Dir;
@ -201,16 +201,9 @@ impl ChunkStore {
}; };
let mut verbose = true; let mut verbose = true;
let mut last_percentage = 0;
Ok((0..0x10000).filter_map(move |index| { Ok((0..0x10000).filter_map(move |index| {
if print_percentage { let percentage = (index * 100) / 0x10000;
let percentage = (index * 100) / 0x10000;
if last_percentage != percentage {
last_percentage = percentage;
eprintln!("percentage done: {}", percentage);
}
}
let subdir: &str = &format!("{:04x}", index); let subdir: &str = &format!("{:04x}", index);
match tools::fs::read_subdir(base_handle.as_raw_fd(), subdir) { match tools::fs::read_subdir(base_handle.as_raw_fd(), subdir) {
Err(e) => { Err(e) => {
@ -220,11 +213,11 @@ impl ChunkStore {
} }
None None
} }
Ok(iter) => Some(iter), Ok(iter) => Some(iter.map(move |item| (item, percentage))),
} }
}) })
.flatten() .flatten()
.filter(|entry| { .filter(|(entry, _percentage)| {
// Check that the file name is actually a hash! (64 hex digits) // Check that the file name is actually a hash! (64 hex digits)
let entry = match entry { let entry = match entry {
Err(_) => return true, // pass errors onwards Err(_) => return true, // pass errors onwards
@ -250,7 +243,8 @@ impl ChunkStore {
pub fn sweep_unused_chunks( pub fn sweep_unused_chunks(
&self, &self,
oldest_writer: Option<i64>, oldest_writer: Option<i64>,
status: &mut GarbageCollectionStatus status: &mut GarbageCollectionStatus,
worker: Arc<WorkerTask>,
) -> Result<(), Error> { ) -> Result<(), Error> {
use nix::sys::stat::fstatat; use nix::sys::stat::fstatat;
@ -266,7 +260,12 @@ impl ChunkStore {
min_atime -= 300; // add 5 mins gap for safety min_atime -= 300; // add 5 mins gap for safety
for entry in self.get_chunk_iterator(true)? { let mut last_percentage = 0;
for (entry, percentage) in self.get_chunk_iterator()? {
if last_percentage != percentage {
last_percentage = percentage;
worker.log(format!("percentage done: {}", percentage));
}
tools::fail_on_shutdown()?; tools::fail_on_shutdown()?;
@ -311,6 +310,7 @@ impl ChunkStore {
} }
drop(lock); drop(lock);
} }
Ok(()) Ok(())
} }

7
src/backup/datastore.rs
View File

@ -78,12 +78,11 @@ impl DataStore {
pub fn get_chunk_iterator( pub fn get_chunk_iterator(
&self, &self,
print_percentage: bool,
) -> Result< ) -> Result<
impl Iterator<Item = Result<tools::fs::ReadDirEntry, Error>>, impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize)>,
Error Error
> { > {
self.chunk_store.get_chunk_iterator(print_percentage) self.chunk_store.get_chunk_iterator()
} }
pub fn create_fixed_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<FixedIndexWriter, Error> { pub fn create_fixed_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<FixedIndexWriter, Error> {
@ -267,7 +266,7 @@ impl DataStore {
self.mark_used_chunks(&mut gc_status)?; self.mark_used_chunks(&mut gc_status)?;
worker.log("Start GC phase2 (sweep unused chunks)"); worker.log("Start GC phase2 (sweep unused chunks)");
self.chunk_store.sweep_unused_chunks(oldest_writer, &mut gc_status)?; self.chunk_store.sweep_unused_chunks(oldest_writer, &mut gc_status, worker.clone())?;
worker.log(&format!("Removed bytes: {}", gc_status.removed_bytes)); worker.log(&format!("Removed bytes: {}", gc_status.removed_bytes));
worker.log(&format!("Removed chunks: {}", gc_status.removed_chunks)); worker.log(&format!("Removed chunks: {}", gc_status.removed_chunks));