From 3d3769830be1dc677d487774617f91983bf1f538 Mon Sep 17 00:00:00 2001 From: Dominik Csapak Date: Fri, 18 Jun 2021 11:29:10 +0200 Subject: [PATCH] tape/helpers/snapshot_reader: sort chunks by inode (per index) sort the chunks we want to backup to tape by inode, to gain some speed on spinning disks. this is done per index, not globally. costs a bit memory, but not too much, about 16 bytes per chunk which would mean ~4MiB for a 1TiB index with 4MiB chunks. Signed-off-by: Dominik Csapak --- src/tape/helpers/snapshot_reader.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/tape/helpers/snapshot_reader.rs b/src/tape/helpers/snapshot_reader.rs index 7b272e37..416c88c1 100644 --- a/src/tape/helpers/snapshot_reader.rs +++ b/src/tape/helpers/snapshot_reader.rs @@ -107,7 +107,7 @@ impl SnapshotReader { pub struct SnapshotChunkIterator<'a> { snapshot_reader: &'a SnapshotReader, todo_list: Vec, - current_index: Option<(Arc>, usize)>, + current_index: Option<(Arc>, usize, Vec<(usize, u64)>)>, } impl <'a> Iterator for SnapshotChunkIterator<'a> { @@ -119,20 +119,26 @@ impl <'a> Iterator for SnapshotChunkIterator<'a> { if self.current_index.is_none() { if let Some(filename) = self.todo_list.pop() { let file = self.snapshot_reader.open_file(&filename)?; - let index: Box = match archive_type(&filename)? { + let index: Box = match archive_type(&filename)? { ArchiveType::FixedIndex => Box::new(FixedIndexReader::new(file)?), ArchiveType::DynamicIndex => Box::new(DynamicIndexReader::new(file)?), _ => bail!("SnapshotChunkIterator: got unknown file type - internal error"), }; - self.current_index = Some((Arc::new(index), 0)); + + let datastore = + DataStore::lookup_datastore(self.snapshot_reader.datastore_name())?; + let order = datastore.get_chunks_in_order(&index, |_| false, |_| Ok(()))?; + + self.current_index = Some((Arc::new(index), 0, order)); } else { return Ok(None); } } - let (index, pos) = self.current_index.take().unwrap(); - if pos < index.index_count() { - let digest = *index.index_digest(pos).unwrap(); - self.current_index = Some((index, pos + 1)); + let (index, pos, list) = self.current_index.take().unwrap(); + if pos < list.len() { + let (real_pos, _) = list[pos]; + let digest = *index.index_digest(real_pos).unwrap(); + self.current_index = Some((index, pos + 1, list)); return Ok(Some(digest)); } else { // pop next index