tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

src/backup/chunk_store.rs - get_chunk_iterator: return percentage inside iterator item

Browse Source
This commit is contained in:
Dietmar Maurer 2019-07-04 09:26:44 +02:00
parent 0f0a35b390
commit a57360983b
2 changed files with 17 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/backup/chunk_store.rs
View File

@ -8,6 +8,7 @@ use serde::Serialize;
use crate::tools;
use super::DataChunk;
use crate::server::WorkerTask;
#[derive(Clone, Serialize)]
pub struct GarbageCollectionStatus {
@ -184,9 +185,8 @@ impl ChunkStore {
pub fn get_chunk_iterator(
&self,
print_percentage: bool,
) -> Result<
impl Iterator<Item = Result<tools::fs::ReadDirEntry, Error>> + std::iter::FusedIterator,
impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize)> + std::iter::FusedIterator,
Error
> {
use nix::dir::Dir;
@ -201,16 +201,9 @@ impl ChunkStore {
};
let mut verbose = true;
let mut last_percentage = 0;
Ok((0..0x10000).filter_map(move |index| {
if print_percentage {
let percentage = (index * 100) / 0x10000;
if last_percentage != percentage {
last_percentage = percentage;
eprintln!("percentage done: {}", percentage);
}
}
let percentage = (index * 100) / 0x10000;
let subdir: &str = &format!("{:04x}", index);
match tools::fs::read_subdir(base_handle.as_raw_fd(), subdir) {
Err(e) => {
@ -220,11 +213,11 @@ impl ChunkStore {
}
None
}
Ok(iter) => Some(iter),
Ok(iter) => Some(iter.map(move |item| (item, percentage))),
}
})
.flatten()
.filter(|entry| {
.filter(|(entry, _percentage)| {
// Check that the file name is actually a hash! (64 hex digits)
let entry = match entry {
Err(_) => return true, // pass errors onwards
@ -250,7 +243,8 @@ impl ChunkStore {
pub fn sweep_unused_chunks(
&self,
oldest_writer: Option<i64>,
status: &mut GarbageCollectionStatus
status: &mut GarbageCollectionStatus,
worker: Arc<WorkerTask>,
) -> Result<(), Error> {
use nix::sys::stat::fstatat;
@ -266,7 +260,12 @@ impl ChunkStore {
min_atime -= 300; // add 5 mins gap for safety
for entry in self.get_chunk_iterator(true)? {
let mut last_percentage = 0;
for (entry, percentage) in self.get_chunk_iterator()? {
if last_percentage != percentage {
last_percentage = percentage;
worker.log(format!("percentage done: {}", percentage));
}
tools::fail_on_shutdown()?;
@ -311,6 +310,7 @@ impl ChunkStore {
}
drop(lock);
}
Ok(())
}

7
src/backup/datastore.rs
View File

@ -78,12 +78,11 @@ impl DataStore {
pub fn get_chunk_iterator(
&self,
print_percentage: bool,
) -> Result<
impl Iterator<Item = Result<tools::fs::ReadDirEntry, Error>>,
impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize)>,
Error
> {
self.chunk_store.get_chunk_iterator(print_percentage)
self.chunk_store.get_chunk_iterator()
}
pub fn create_fixed_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<FixedIndexWriter, Error> {
@ -267,7 +266,7 @@ impl DataStore {
self.mark_used_chunks(&mut gc_status)?;
worker.log("Start GC phase2 (sweep unused chunks)");
self.chunk_store.sweep_unused_chunks(oldest_writer, &mut gc_status)?;
self.chunk_store.sweep_unused_chunks(oldest_writer, &mut gc_status, worker.clone())?;
worker.log(&format!("Removed bytes: {}", gc_status.removed_bytes));
worker.log(&format!("Removed chunks: {}", gc_status.removed_chunks));