diff --git a/Cargo.toml b/Cargo.toml index 2414fcf9..4e299d41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,6 @@ tokio-net = { version = "0.2.0-alpha.4", features = ["signal"] } tokio-openssl = "0.4.0-alpha.2" tower-service = "0.3.0-alpha.1" url = "1.7" -uuid = { version = "0.7", features = ["v4"] } valgrind_request = { version = "1.1", optional = true } walkdir = "2" xdg = "2.2" diff --git a/debian/control b/debian/control index d17c620c..60fc5dbe 100644 --- a/debian/control +++ b/debian/control @@ -5,14 +5,15 @@ Maintainer: Proxmox Support Team Build-Depends: bash-completion, clang, debhelper (>= 10), + latexmk, libfuse3-dev, libpam0g-dev, libsystemd-dev, + libuuid1, libzstd-dev (>= 1.3.8), pkg-config, python3-docutils, python3-sphinx, - latexmk, Standards-Version: 3.9.5 Homepage: https://www.proxmox.com diff --git a/src/backup/dynamic_index.rs b/src/backup/dynamic_index.rs index eb780c7a..af848e93 100644 --- a/src/backup/dynamic_index.rs +++ b/src/backup/dynamic_index.rs @@ -6,16 +6,16 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use failure::*; -use uuid::Uuid; use proxmox::tools::io::ReadExt; +use proxmox::tools::uuid::Uuid; use proxmox::tools::vec; -use super::Chunker; -use super::IndexFile; use super::chunk_stat::ChunkStat; use super::chunk_store::ChunkStore; use super::read_chunk::ReadChunk; +use super::Chunker; +use super::IndexFile; use super::{DataBlob, DataChunkBuilder}; use crate::tools; @@ -52,7 +52,6 @@ unsafe impl Send for DynamicIndexReader {} unsafe impl Sync for DynamicIndexReader {} impl Drop for DynamicIndexReader { - fn drop(&mut self) { if let Err(err) = self.unmap() { eprintln!("Unable to unmap dynamic index - {}", err); @@ -61,9 +60,7 @@ impl Drop for DynamicIndexReader { } impl DynamicIndexReader { - pub fn open(path: &Path) -> Result { - File::open(path) .map_err(Error::from) .and_then(|file| Self::new(file)) @@ -71,8 +68,9 @@ impl DynamicIndexReader { } pub fn new(mut file: std::fs::File) -> Result { - - if let Err(err) = nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock) { + if let Err(err) = + nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock) + { bail!("unable to get shared lock - {}", err); } @@ -99,19 +97,22 @@ impl DynamicIndexReader { bail!("got unexpected file size"); } - let data = unsafe { nix::sys::mman::mmap( - std::ptr::null_mut(), - index_size, - nix::sys::mman::ProtFlags::PROT_READ, - nix::sys::mman::MapFlags::MAP_PRIVATE, - rawfd, - header_size as i64) }? as *const u8; + let data = unsafe { + nix::sys::mman::mmap( + std::ptr::null_mut(), + index_size, + nix::sys::mman::ProtFlags::PROT_READ, + nix::sys::mman::MapFlags::MAP_PRIVATE, + rawfd, + header_size as i64, + ) + }? as *const u8; Ok(Self { _file: file, size, index: data, - index_entries: index_size/40, + index_entries: index_size / 40, ctime, uuid: header.uuid, index_csum: header.index_csum, @@ -119,10 +120,13 @@ impl DynamicIndexReader { } fn unmap(&mut self) -> Result<(), Error> { + if self.index == std::ptr::null_mut() { + return Ok(()); + } - if self.index == std::ptr::null_mut() { return Ok(()); } - - if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, self.index_entries*40) } { + if let Err(err) = unsafe { + nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, self.index_entries * 40) + } { bail!("unmap dynamic index failed - {}", err); } @@ -139,15 +143,15 @@ impl DynamicIndexReader { let start = if pos == 0 { 0 } else { - unsafe { *(self.index.add((pos-1)*40) as *const u64) } + unsafe { *(self.index.add((pos - 1) * 40) as *const u64) } }; - let end = unsafe { *(self.index.add(pos*40) as *const u64) }; + let end = unsafe { *(self.index.add(pos * 40) as *const u64) }; let mut digest = std::mem::MaybeUninit::<[u8; 32]>::uninit(); unsafe { std::ptr::copy_nonoverlapping( - self.index.add(pos*40+8), + self.index.add(pos * 40 + 8), (*digest.as_mut_ptr()).as_mut_ptr(), 32, ); @@ -162,7 +166,7 @@ impl DynamicIndexReader { if pos >= self.index_entries { panic!("chunk index out of range"); } - unsafe { *(self.index.add(pos*40) as *const u64) } + unsafe { *(self.index.add(pos * 40) as *const u64) } } #[inline] @@ -170,13 +174,12 @@ impl DynamicIndexReader { if pos >= self.index_entries { panic!("chunk index out of range"); } - let slice = unsafe { std::slice::from_raw_parts(self.index.add(pos*40+8), 32) }; + let slice = unsafe { std::slice::from_raw_parts(self.index.add(pos * 40 + 8), 32) }; slice.try_into().unwrap() } /// Compute checksum and data size pub fn compute_csum(&self) -> ([u8; 32], u64) { - let mut csum = openssl::sha::Sha256::new(); let mut chunk_end = 0; for pos in 0..self.index_entries { @@ -213,9 +216,8 @@ impl DynamicIndexReader { start: u64, end_idx: usize, end: u64, - offset: u64 + offset: u64, ) -> Result { - if (offset >= end) || (offset < start) { bail!("offset out of range"); } @@ -223,7 +225,7 @@ impl DynamicIndexReader { if end_idx == start_idx { return Ok(start_idx); // found } - let middle_idx = (start_idx + end_idx)/2; + let middle_idx = (start_idx + end_idx) / 2; let middle_end = self.chunk_end(middle_idx); if offset < middle_end { @@ -243,9 +245,7 @@ impl IndexFile for DynamicIndexReader { if pos >= self.index_entries { None } else { - Some(unsafe { - std::mem::transmute(self.chunk_digest(pos).as_ptr()) - }) + Some(unsafe { std::mem::transmute(self.chunk_digest(pos).as_ptr()) }) } } @@ -268,26 +268,25 @@ pub struct BufferedDynamicReader { read_offset: u64, } -impl BufferedDynamicReader { - +impl BufferedDynamicReader { pub fn new(index: DynamicIndexReader, store: S) -> Self { - let archive_size = index.chunk_end(index.index_entries - 1); Self { store, index, archive_size, - read_buffer: Vec::with_capacity(1024*1024), + read_buffer: Vec::with_capacity(1024 * 1024), buffered_chunk_idx: 0, buffered_chunk_start: 0, read_offset: 0, } } - pub fn archive_size(&self) -> u64 { self.archive_size } + pub fn archive_size(&self) -> u64 { + self.archive_size + } fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> { - let index = &self.index; let (start, end, digest) = index.chunk_info(idx)?; @@ -295,8 +294,12 @@ impl BufferedDynamicReader { let data = self.store.read_chunk(&digest)?; - if (end - start) != data.len() as u64 { - bail!("read chunk with wrong size ({} != {}", (end - start), data.len()); + if (end - start) != data.len() as u64 { + bail!( + "read chunk with wrong size ({} != {}", + (end - start), + data.len() + ); } self.read_buffer.clear(); @@ -310,19 +313,19 @@ impl BufferedDynamicReader { } } -impl crate::tools::BufferedRead for BufferedDynamicReader { - +impl crate::tools::BufferedRead for BufferedDynamicReader { fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error> { - - if offset == self.archive_size { return Ok(&self.read_buffer[0..0]); } + if offset == self.archive_size { + return Ok(&self.read_buffer[0..0]); + } let buffer_len = self.read_buffer.len(); let index = &self.index; // optimization for sequential read - if buffer_len > 0 && - ((self.buffered_chunk_idx + 1) < index.index_entries) && - (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64))) + if buffer_len > 0 + && ((self.buffered_chunk_idx + 1) < index.index_entries) + && (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64))) { let next_idx = self.buffered_chunk_idx + 1; let next_end = index.chunk_end(next_idx); @@ -333,36 +336,40 @@ impl crate::tools::BufferedRead for BufferedDynamicReader { } } - if (buffer_len == 0) || - (offset < self.buffered_chunk_start) || - (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64))) + if (buffer_len == 0) + || (offset < self.buffered_chunk_start) + || (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64))) { let end_idx = index.index_entries - 1; let end = index.chunk_end(end_idx); let idx = index.binary_search(0, 0, end_idx, end, offset)?; self.buffer_chunk(idx)?; - } + } let buffer_offset = (offset - self.buffered_chunk_start) as usize; Ok(&self.read_buffer[buffer_offset..]) } } -impl std::io::Read for BufferedDynamicReader { - +impl std::io::Read for BufferedDynamicReader { fn read(&mut self, buf: &mut [u8]) -> Result { - - use std::io::{Error, ErrorKind}; use crate::tools::BufferedRead; + use std::io::{Error, ErrorKind}; let data = match self.buffered_read(self.read_offset) { Ok(v) => v, Err(err) => return Err(Error::new(ErrorKind::Other, err.to_string())), }; - let n = if data.len() > buf.len() { buf.len() } else { data.len() }; + let n = if data.len() > buf.len() { + buf.len() + } else { + data.len() + }; - unsafe { std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n); } + unsafe { + std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n); + } self.read_offset += n as u64; @@ -370,13 +377,11 @@ impl std::io::Read for BufferedDynamicReader { } } -impl std::io::Seek for BufferedDynamicReader { - +impl std::io::Seek for BufferedDynamicReader { fn seek(&mut self, pos: SeekFrom) -> Result { - let new_offset = match pos { - SeekFrom::Start(start_offset) => start_offset as i64, - SeekFrom::End(end_offset) => (self.archive_size as i64)+ end_offset, + SeekFrom::Start(start_offset) => start_offset as i64, + SeekFrom::End(end_offset) => (self.archive_size as i64) + end_offset, SeekFrom::Current(offset) => (self.read_offset as i64) + offset, }; @@ -384,7 +389,11 @@ impl std::io::Seek for BufferedDynamicReader { if (new_offset < 0) || (new_offset > (self.archive_size as i64)) { return Err(Error::new( ErrorKind::Other, - format!("seek is out of range {} ([0..{}])", new_offset, self.archive_size))); + format!( + "seek is out of range {} ([0..{}])", + new_offset, self.archive_size + ), + )); } self.read_offset = new_offset as u64; @@ -406,16 +415,13 @@ pub struct DynamicIndexWriter { } impl Drop for DynamicIndexWriter { - fn drop(&mut self) { let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors } } impl DynamicIndexWriter { - pub fn create(store: Arc, path: &Path) -> Result { - let shared_lock = store.try_shared_lock()?; let full_path = store.relative_path(path); @@ -423,22 +429,26 @@ impl DynamicIndexWriter { tmp_path.set_extension("tmp_didx"); let file = std::fs::OpenOptions::new() - .create(true).truncate(true) + .create(true) + .truncate(true) .read(true) .write(true) .open(&tmp_path)?; - let mut writer = BufWriter::with_capacity(1024*1024, file); + let mut writer = BufWriter::with_capacity(1024 * 1024, file); let header_size = std::mem::size_of::(); // todo: use static assertion when available in rust - if header_size != 4096 { panic!("got unexpected header size"); } + if header_size != 4096 { + panic!("got unexpected header size"); + } - let ctime = std::time::SystemTime::now().duration_since( - std::time::SystemTime::UNIX_EPOCH)?.as_secs(); + let ctime = std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH)? + .as_secs(); - let uuid = Uuid::new_v4(); + let uuid = Uuid::generate(); let mut buffer = vec::zeroed(header_size); let header = crate::tools::map_struct_mut::(&mut buffer)?; @@ -471,10 +481,12 @@ impl DynamicIndexWriter { self.store.insert_chunk(chunk, digest) } - pub fn close(&mut self) -> Result<[u8; 32], Error> { - + pub fn close(&mut self) -> Result<[u8; 32], Error> { if self.closed { - bail!("cannot close already closed archive index file {:?}", self.filename); + bail!( + "cannot close already closed archive index file {:?}", + self.filename + ); } self.closed = true; @@ -490,7 +502,6 @@ impl DynamicIndexWriter { self.writer.write_all(&index_csum)?; self.writer.flush()?; - if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) { bail!("Atomic rename file {:?} failed - {}", self.filename, err); } @@ -501,10 +512,13 @@ impl DynamicIndexWriter { // fixme: rename to add_digest pub fn add_chunk(&mut self, offset: u64, digest: &[u8; 32]) -> Result<(), Error> { if self.closed { - bail!("cannot write to closed dynamic index file {:?}", self.filename); + bail!( + "cannot write to closed dynamic index file {:?}", + self.filename + ); } - let offset_le: &[u8; 8] = unsafe { &std::mem::transmute::(offset.to_le()) }; + let offset_le: &[u8; 8] = unsafe { &std::mem::transmute::(offset.to_le()) }; if let Some(ref mut csum) = self.csum { csum.update(offset_le); @@ -531,7 +545,6 @@ pub struct DynamicChunkWriter { } impl DynamicChunkWriter { - pub fn new(index: DynamicIndexWriter, chunk_size: usize) -> Self { Self { index, @@ -540,7 +553,7 @@ impl DynamicChunkWriter { stat: ChunkStat::new(0), chunk_offset: 0, last_chunk: 0, - chunk_buffer: Vec::with_capacity(chunk_size*4), + chunk_buffer: Vec::with_capacity(chunk_size * 4), } } @@ -548,8 +561,7 @@ impl DynamicChunkWriter { &self.stat } - pub fn close(&mut self) -> Result<(), Error> { - + pub fn close(&mut self) -> Result<(), Error> { if self.closed { return Ok(()); } @@ -563,16 +575,18 @@ impl DynamicChunkWriter { self.stat.size = self.chunk_offset as u64; // add size of index file - self.stat.size += (self.stat.chunk_count*40 + std::mem::size_of::()) as u64; + self.stat.size += + (self.stat.chunk_count * 40 + std::mem::size_of::()) as u64; Ok(()) } fn write_chunk_buffer(&mut self) -> Result<(), Error> { - let chunk_size = self.chunk_buffer.len(); - if chunk_size == 0 { return Ok(()); } + if chunk_size == 0 { + return Ok(()); + } let expected_chunk_size = self.chunk_offset - self.last_chunk; if expected_chunk_size != self.chunk_buffer.len() { @@ -589,7 +603,6 @@ impl DynamicChunkWriter { match self.index.insert_chunk(&chunk, &digest) { Ok((is_duplicate, compressed_size)) => { - self.stat.compressed_size += compressed_size; if is_duplicate { self.stat.duplicate_chunks += 1; @@ -597,8 +610,14 @@ impl DynamicChunkWriter { self.stat.disk_size += compressed_size; } - println!("ADD CHUNK {:016x} {} {}% {} {}", self.chunk_offset, chunk_size, - (compressed_size*100)/(chunk_size as u64), is_duplicate, proxmox::tools::digest_to_hex(&digest)); + println!( + "ADD CHUNK {:016x} {} {}% {} {}", + self.chunk_offset, + chunk_size, + (compressed_size * 100) / (chunk_size as u64), + is_duplicate, + proxmox::tools::digest_to_hex(&digest) + ); self.index.add_chunk(self.chunk_offset as u64, &digest)?; self.chunk_buffer.truncate(0); Ok(()) @@ -612,9 +631,7 @@ impl DynamicChunkWriter { } impl Write for DynamicChunkWriter { - fn write(&mut self, data: &[u8]) -> std::result::Result { - let chunker = &mut self.chunker; let pos = chunker.scan(data); @@ -624,10 +641,12 @@ impl Write for DynamicChunkWriter { self.chunk_offset += pos; if let Err(err) = self.write_chunk_buffer() { - return Err(std::io::Error::new(std::io::ErrorKind::Other, err.to_string())); + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + err.to_string(), + )); } Ok(pos) - } else { self.chunk_offset += data.len(); self.chunk_buffer.extend_from_slice(data); @@ -636,6 +655,9 @@ impl Write for DynamicChunkWriter { } fn flush(&mut self) -> std::result::Result<(), std::io::Error> { - Err(std::io::Error::new(std::io::ErrorKind::Other, "please use close() instead of flush()")) + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "please use close() instead of flush()", + )) } } diff --git a/src/backup/fixed_index.rs b/src/backup/fixed_index.rs index 991f621a..db0c52d0 100644 --- a/src/backup/fixed_index.rs +++ b/src/backup/fixed_index.rs @@ -1,24 +1,24 @@ use failure::*; -use std::io::{Seek, SeekFrom}; use std::convert::TryInto; +use std::io::{Seek, SeekFrom}; -use crate::tools; -use super::IndexFile; use super::chunk_stat::*; use super::chunk_store::*; +use super::IndexFile; +use crate::tools; -use std::sync::Arc; -use std::io::Write; -use std::fs::File; -use std::path::{Path, PathBuf}; -use std::os::unix::io::AsRawFd; -use uuid::Uuid; use chrono::{Local, TimeZone}; +use std::fs::File; +use std::io::Write; +use std::os::unix::io::AsRawFd; +use std::path::{Path, PathBuf}; +use std::sync::Arc; -use super::ChunkInfo; use super::read_chunk::*; +use super::ChunkInfo; use proxmox::tools::io::ReadExt; +use proxmox::tools::Uuid; /// Header format definition for fixed index files (`.fidx`) #[repr(C)] @@ -52,7 +52,6 @@ unsafe impl Send for FixedIndexReader {} unsafe impl Sync for FixedIndexReader {} impl Drop for FixedIndexReader { - fn drop(&mut self) { if let Err(err) = self.unmap() { eprintln!("Unable to unmap file - {}", err); @@ -61,9 +60,7 @@ impl Drop for FixedIndexReader { } impl FixedIndexReader { - pub fn open(path: &Path) -> Result { - File::open(path) .map_err(Error::from) .and_then(|file| Self::new(file)) @@ -71,8 +68,9 @@ impl FixedIndexReader { } pub fn new(mut file: std::fs::File) -> Result { - - if let Err(err) = nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock) { + if let Err(err) = + nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock) + { bail!("unable to get shared lock - {}", err); } @@ -89,8 +87,8 @@ impl FixedIndexReader { let ctime = u64::from_le(header.ctime); let chunk_size = u64::from_le(header.chunk_size); - let index_length = ((size + chunk_size - 1)/chunk_size) as usize; - let index_size = index_length*32; + let index_length = ((size + chunk_size - 1) / chunk_size) as usize; + let index_size = index_length * 32; let rawfd = file.as_raw_fd(); @@ -101,16 +99,23 @@ impl FixedIndexReader { let expected_index_size = (stat.st_size as usize) - header_size; if index_size != expected_index_size { - bail!("got unexpected file size ({} != {})", index_size, expected_index_size); + bail!( + "got unexpected file size ({} != {})", + index_size, + expected_index_size + ); } - let data = unsafe { nix::sys::mman::mmap( - std::ptr::null_mut(), - index_size, - nix::sys::mman::ProtFlags::PROT_READ, - nix::sys::mman::MapFlags::MAP_PRIVATE, - file.as_raw_fd(), - header_size as i64) }? as *mut u8; + let data = unsafe { + nix::sys::mman::mmap( + std::ptr::null_mut(), + index_size, + nix::sys::mman::ProtFlags::PROT_READ, + nix::sys::mman::MapFlags::MAP_PRIVATE, + file.as_raw_fd(), + header_size as i64, + ) + }? as *mut u8; Ok(Self { _file: file, @@ -125,12 +130,15 @@ impl FixedIndexReader { } fn unmap(&mut self) -> Result<(), Error> { + if self.index == std::ptr::null_mut() { + return Ok(()); + } - if self.index == std::ptr::null_mut() { return Ok(()); } + let index_size = self.index_length * 32; - let index_size = self.index_length*32; - - if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } { + if let Err(err) = + unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } + { bail!("unmap file failed - {}", err); } @@ -140,7 +148,6 @@ impl FixedIndexReader { } pub fn chunk_info(&self, pos: usize) -> Result<(u64, u64, [u8; 32]), Error> { - if pos >= self.index_length { bail!("chunk index out of range"); } @@ -154,7 +161,7 @@ impl FixedIndexReader { let mut digest = std::mem::MaybeUninit::<[u8; 32]>::uninit(); unsafe { std::ptr::copy_nonoverlapping( - self.index.add(pos*32), + self.index.add(pos * 32), (*digest.as_mut_ptr()).as_mut_ptr(), 32, ); @@ -168,7 +175,7 @@ impl FixedIndexReader { if pos >= self.index_length { panic!("chunk index out of range"); } - let slice = unsafe { std::slice::from_raw_parts(self.index.add(pos*32), 32) }; + let slice = unsafe { std::slice::from_raw_parts(self.index.add(pos * 32), 32) }; slice.try_into().unwrap() } @@ -178,7 +185,7 @@ impl FixedIndexReader { panic!("chunk index out of range"); } - let end = ((pos+1) * self.chunk_size) as u64; + let end = ((pos + 1) * self.chunk_size) as u64; if end > self.size { self.size } else { @@ -188,11 +195,10 @@ impl FixedIndexReader { /// Compute checksum and data size pub fn compute_csum(&self) -> ([u8; 32], u64) { - let mut csum = openssl::sha::Sha256::new(); let mut chunk_end = 0; for pos in 0..self.index_length { - chunk_end = ((pos+1) * self.chunk_size) as u64; + chunk_end = ((pos + 1) * self.chunk_size) as u64; let digest = self.chunk_digest(pos); csum.update(digest); } @@ -204,7 +210,10 @@ impl FixedIndexReader { pub fn print_info(&self) { println!("Size: {}", self.size); println!("ChunkSize: {}", self.chunk_size); - println!("CTime: {}", Local.timestamp(self.ctime as i64, 0).format("%c")); + println!( + "CTime: {}", + Local.timestamp(self.ctime as i64, 0).format("%c") + ); println!("UUID: {:?}", self.uuid); } } @@ -218,7 +227,7 @@ impl IndexFile for FixedIndexReader { if pos >= self.index_length { None } else { - Some(unsafe { std::mem::transmute(self.index.add(pos*32)) }) + Some(unsafe { std::mem::transmute(self.index.add(pos * 32)) }) } } @@ -245,7 +254,6 @@ pub struct FixedIndexWriter { unsafe impl Send for FixedIndexWriter {} impl Drop for FixedIndexWriter { - fn drop(&mut self) { let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors if let Err(err) = self.unmap() { @@ -255,10 +263,13 @@ impl Drop for FixedIndexWriter { } impl FixedIndexWriter { - #[allow(clippy::cast_ptr_alignment)] - pub fn create(store: Arc, path: &Path, size: usize, chunk_size: usize) -> Result { - + pub fn create( + store: Arc, + path: &Path, + size: usize, + chunk_size: usize, + ) -> Result { let shared_lock = store.try_shared_lock()?; let full_path = store.relative_path(path); @@ -266,7 +277,8 @@ impl FixedIndexWriter { tmp_path.set_extension("tmp_fidx"); let mut file = std::fs::OpenOptions::new() - .create(true).truncate(true) + .create(true) + .truncate(true) .read(true) .write(true) .open(&tmp_path)?; @@ -274,15 +286,18 @@ impl FixedIndexWriter { let header_size = std::mem::size_of::(); // todo: use static assertion when available in rust - if header_size != 4096 { panic!("got unexpected header size"); } + if header_size != 4096 { + panic!("got unexpected header size"); + } - let ctime = std::time::SystemTime::now().duration_since( - std::time::SystemTime::UNIX_EPOCH)?.as_secs(); + let ctime = std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH)? + .as_secs(); - let uuid = Uuid::new_v4(); + let uuid = Uuid::generate(); let buffer = vec![0u8; header_size]; - let header = unsafe { &mut * (buffer.as_ptr() as *mut FixedIndexHeader) }; + let header = unsafe { &mut *(buffer.as_ptr() as *mut FixedIndexHeader) }; header.magic = super::FIXED_SIZED_CHUNK_INDEX_1_0; header.ctime = u64::to_le(ctime); @@ -294,17 +309,20 @@ impl FixedIndexWriter { file.write_all(&buffer)?; - let index_length = (size + chunk_size - 1)/chunk_size; - let index_size = index_length*32; + let index_length = (size + chunk_size - 1) / chunk_size; + let index_size = index_length * 32; nix::unistd::ftruncate(file.as_raw_fd(), (header_size + index_size) as i64)?; - let data = unsafe { nix::sys::mman::mmap( - std::ptr::null_mut(), - index_size, - nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE, - nix::sys::mman::MapFlags::MAP_SHARED, - file.as_raw_fd(), - header_size as i64) }? as *mut u8; + let data = unsafe { + nix::sys::mman::mmap( + std::ptr::null_mut(), + index_size, + nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE, + nix::sys::mman::MapFlags::MAP_SHARED, + file.as_raw_fd(), + header_size as i64, + ) + }? as *mut u8; Ok(Self { store, @@ -326,12 +344,15 @@ impl FixedIndexWriter { } fn unmap(&mut self) -> Result<(), Error> { + if self.index == std::ptr::null_mut() { + return Ok(()); + } - if self.index == std::ptr::null_mut() { return Ok(()); } + let index_size = self.index_length * 32; - let index_size = self.index_length*32; - - if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } { + if let Err(err) = + unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } + { bail!("unmap file {:?} failed - {}", self.tmp_filename, err); } @@ -340,13 +361,14 @@ impl FixedIndexWriter { Ok(()) } - pub fn close(&mut self) -> Result<[u8; 32], Error> { + pub fn close(&mut self) -> Result<[u8; 32], Error> { + if self.index == std::ptr::null_mut() { + bail!("cannot close already closed index file."); + } - if self.index == std::ptr::null_mut() { bail!("cannot close already closed index file."); } - - let index_size = self.index_length*32; + let index_size = self.index_length * 32; let data = unsafe { std::slice::from_raw_parts(self.index, index_size) }; - let index_csum = openssl::sha::sha256(data); + let index_csum = openssl::sha::sha256(data); self.unmap()?; @@ -363,7 +385,6 @@ impl FixedIndexWriter { } pub fn check_chunk_alignment(&self, offset: usize, chunk_len: usize) -> Result { - if offset < chunk_len { bail!("got chunk with small offset ({} < {}", offset, chunk_len); } @@ -375,12 +396,18 @@ impl FixedIndexWriter { } // last chunk can be smaller - if ((offset != self.size) && (chunk_len != self.chunk_size)) || - (chunk_len > self.chunk_size) || (chunk_len == 0) { - bail!("chunk with unexpected length ({} != {}", chunk_len, self.chunk_size); - } + if ((offset != self.size) && (chunk_len != self.chunk_size)) + || (chunk_len > self.chunk_size) + || (chunk_len == 0) + { + bail!( + "chunk with unexpected length ({} != {}", + chunk_len, + self.chunk_size + ); + } - if pos & (self.chunk_size-1) != 0 { + if pos & (self.chunk_size - 1) != 0 { bail!("got unaligned chunk (pos = {})", pos); } @@ -389,22 +416,28 @@ impl FixedIndexWriter { // Note: We want to add data out of order, so do not assume any order here. pub fn add_chunk(&mut self, chunk_info: &ChunkInfo, stat: &mut ChunkStat) -> Result<(), Error> { - let chunk_len = chunk_info.chunk_len as usize; let offset = chunk_info.offset as usize; // end of chunk let idx = self.check_chunk_alignment(offset, chunk_len)?; - let (is_duplicate, compressed_size) = - self.store.insert_chunk(&chunk_info.chunk, &chunk_info.digest)?; + let (is_duplicate, compressed_size) = self + .store + .insert_chunk(&chunk_info.chunk, &chunk_info.digest)?; stat.chunk_count += 1; stat.compressed_size += compressed_size; let digest = &chunk_info.digest; - println!("ADD CHUNK {} {} {}% {} {}", idx, chunk_len, - (compressed_size*100)/(chunk_len as u64), is_duplicate, proxmox::tools::digest_to_hex(digest)); + println!( + "ADD CHUNK {} {} {}% {} {}", + idx, + chunk_len, + (compressed_size * 100) / (chunk_len as u64), + is_duplicate, + proxmox::tools::digest_to_hex(digest) + ); if is_duplicate { stat.duplicate_chunks += 1; @@ -416,14 +449,19 @@ impl FixedIndexWriter { } pub fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> { - if index >= self.index_length { - bail!("add digest failed - index out of range ({} >= {})", index, self.index_length); + bail!( + "add digest failed - index out of range ({} >= {})", + index, + self.index_length + ); } - if self.index == std::ptr::null_mut() { bail!("cannot write to closed index file."); } + if self.index == std::ptr::null_mut() { + bail!("cannot write to closed index file."); + } - let index_pos = index*32; + let index_pos = index * 32; unsafe { let dst = self.index.add(index_pos); dst.copy_from_nonoverlapping(digest.as_ptr(), 32); @@ -443,26 +481,25 @@ pub struct BufferedFixedReader { read_offset: u64, } -impl BufferedFixedReader { - +impl BufferedFixedReader { pub fn new(index: FixedIndexReader, store: S) -> Self { - let archive_size = index.size; Self { store, index, archive_size, - read_buffer: Vec::with_capacity(1024*1024), + read_buffer: Vec::with_capacity(1024 * 1024), buffered_chunk_idx: 0, buffered_chunk_start: 0, read_offset: 0, } } - pub fn archive_size(&self) -> u64 { self.archive_size } + pub fn archive_size(&self) -> u64 { + self.archive_size + } fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> { - let index = &self.index; let (start, end, digest) = index.chunk_info(idx)?; @@ -470,8 +507,12 @@ impl BufferedFixedReader { let data = self.store.read_chunk(&digest)?; - if (end - start) != data.len() as u64 { - bail!("read chunk with wrong size ({} != {}", (end - start), data.len()); + if (end - start) != data.len() as u64 { + bail!( + "read chunk with wrong size ({} != {}", + (end - start), + data.len() + ); } self.read_buffer.clear(); @@ -485,19 +526,19 @@ impl BufferedFixedReader { } } -impl crate::tools::BufferedRead for BufferedFixedReader { - +impl crate::tools::BufferedRead for BufferedFixedReader { fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error> { - - if offset == self.archive_size { return Ok(&self.read_buffer[0..0]); } + if offset == self.archive_size { + return Ok(&self.read_buffer[0..0]); + } let buffer_len = self.read_buffer.len(); let index = &self.index; // optimization for sequential read - if buffer_len > 0 && - ((self.buffered_chunk_idx + 1) < index.index_length) && - (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64))) + if buffer_len > 0 + && ((self.buffered_chunk_idx + 1) < index.index_length) + && (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64))) { let next_idx = self.buffered_chunk_idx + 1; let next_end = index.chunk_end(next_idx); @@ -508,34 +549,38 @@ impl crate::tools::BufferedRead for BufferedFixedReader { } } - if (buffer_len == 0) || - (offset < self.buffered_chunk_start) || - (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64))) + if (buffer_len == 0) + || (offset < self.buffered_chunk_start) + || (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64))) { let idx = (offset / index.chunk_size as u64) as usize; self.buffer_chunk(idx)?; - } + } let buffer_offset = (offset - self.buffered_chunk_start) as usize; Ok(&self.read_buffer[buffer_offset..]) } } -impl std::io::Read for BufferedFixedReader { - +impl std::io::Read for BufferedFixedReader { fn read(&mut self, buf: &mut [u8]) -> Result { - - use std::io::{Error, ErrorKind}; use crate::tools::BufferedRead; + use std::io::{Error, ErrorKind}; let data = match self.buffered_read(self.read_offset) { Ok(v) => v, Err(err) => return Err(Error::new(ErrorKind::Other, err.to_string())), }; - let n = if data.len() > buf.len() { buf.len() } else { data.len() }; + let n = if data.len() > buf.len() { + buf.len() + } else { + data.len() + }; - unsafe { std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n); } + unsafe { + std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n); + } self.read_offset += n as u64; @@ -543,13 +588,11 @@ impl std::io::Read for BufferedFixedReader { } } -impl Seek for BufferedFixedReader { - +impl Seek for BufferedFixedReader { fn seek(&mut self, pos: SeekFrom) -> Result { - let new_offset = match pos { - SeekFrom::Start(start_offset) => start_offset as i64, - SeekFrom::End(end_offset) => (self.archive_size as i64)+ end_offset, + SeekFrom::Start(start_offset) => start_offset as i64, + SeekFrom::End(end_offset) => (self.archive_size as i64) + end_offset, SeekFrom::Current(offset) => (self.read_offset as i64) + offset, }; @@ -557,7 +600,11 @@ impl Seek for BufferedFixedReader { if (new_offset < 0) || (new_offset > (self.archive_size as i64)) { return Err(Error::new( ErrorKind::Other, - format!("seek is out of range {} ([0..{}])", new_offset, self.archive_size))); + format!( + "seek is out of range {} ([0..{}])", + new_offset, self.archive_size + ), + )); } self.read_offset = new_offset as u64;