drop uuid crate dependency
proxmox::tools now has a Uuid module using the native libuuid. Adds build dependency: libuuid1 (which is a Pre-Depends of util-linux, so always installed anyway). Drops uuid + 16 more crate dependencies. Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller
parent
58ea88c8ab
commit
f569acc5e2
|
|
@ -44,7 +44,6 @@ tokio-net = { version = "0.2.0-alpha.4", features = ["signal"] }
|
|||
tokio-openssl = "0.4.0-alpha.2"
|
||||
tower-service = "0.3.0-alpha.1"
|
||||
url = "1.7"
|
||||
uuid = { version = "0.7", features = ["v4"] }
|
||||
valgrind_request = { version = "1.1", optional = true }
|
||||
walkdir = "2"
|
||||
xdg = "2.2"
|
||||
|
|
|
|||
|
|
@ -5,14 +5,15 @@ Maintainer: Proxmox Support Team <support@proxmox.com>
|
|||
Build-Depends: bash-completion,
|
||||
clang,
|
||||
debhelper (>= 10),
|
||||
latexmk,
|
||||
libfuse3-dev,
|
||||
libpam0g-dev,
|
||||
libsystemd-dev,
|
||||
libuuid1,
|
||||
libzstd-dev (>= 1.3.8),
|
||||
pkg-config,
|
||||
python3-docutils,
|
||||
python3-sphinx,
|
||||
latexmk,
|
||||
Standards-Version: 3.9.5
|
||||
Homepage: https://www.proxmox.com
|
||||
|
||||
|
|
|
|||
|
|
@ -6,16 +6,16 @@ use std::path::{Path, PathBuf};
|
|||
use std::sync::Arc;
|
||||
|
||||
use failure::*;
|
||||
use uuid::Uuid;
|
||||
|
||||
use proxmox::tools::io::ReadExt;
|
||||
use proxmox::tools::uuid::Uuid;
|
||||
use proxmox::tools::vec;
|
||||
|
||||
use super::Chunker;
|
||||
use super::IndexFile;
|
||||
use super::chunk_stat::ChunkStat;
|
||||
use super::chunk_store::ChunkStore;
|
||||
use super::read_chunk::ReadChunk;
|
||||
use super::Chunker;
|
||||
use super::IndexFile;
|
||||
use super::{DataBlob, DataChunkBuilder};
|
||||
use crate::tools;
|
||||
|
||||
|
|
@ -52,7 +52,6 @@ unsafe impl Send for DynamicIndexReader {}
|
|||
unsafe impl Sync for DynamicIndexReader {}
|
||||
|
||||
impl Drop for DynamicIndexReader {
|
||||
|
||||
fn drop(&mut self) {
|
||||
if let Err(err) = self.unmap() {
|
||||
eprintln!("Unable to unmap dynamic index - {}", err);
|
||||
|
|
@ -61,9 +60,7 @@ impl Drop for DynamicIndexReader {
|
|||
}
|
||||
|
||||
impl DynamicIndexReader {
|
||||
|
||||
pub fn open(path: &Path) -> Result<Self, Error> {
|
||||
|
||||
File::open(path)
|
||||
.map_err(Error::from)
|
||||
.and_then(|file| Self::new(file))
|
||||
|
|
@ -71,8 +68,9 @@ impl DynamicIndexReader {
|
|||
}
|
||||
|
||||
pub fn new(mut file: std::fs::File) -> Result<Self, Error> {
|
||||
|
||||
if let Err(err) = nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock) {
|
||||
if let Err(err) =
|
||||
nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock)
|
||||
{
|
||||
bail!("unable to get shared lock - {}", err);
|
||||
}
|
||||
|
||||
|
|
@ -99,13 +97,16 @@ impl DynamicIndexReader {
|
|||
bail!("got unexpected file size");
|
||||
}
|
||||
|
||||
let data = unsafe { nix::sys::mman::mmap(
|
||||
let data = unsafe {
|
||||
nix::sys::mman::mmap(
|
||||
std::ptr::null_mut(),
|
||||
index_size,
|
||||
nix::sys::mman::ProtFlags::PROT_READ,
|
||||
nix::sys::mman::MapFlags::MAP_PRIVATE,
|
||||
rawfd,
|
||||
header_size as i64) }? as *const u8;
|
||||
header_size as i64,
|
||||
)
|
||||
}? as *const u8;
|
||||
|
||||
Ok(Self {
|
||||
_file: file,
|
||||
|
|
@ -119,10 +120,13 @@ impl DynamicIndexReader {
|
|||
}
|
||||
|
||||
fn unmap(&mut self) -> Result<(), Error> {
|
||||
if self.index == std::ptr::null_mut() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if self.index == std::ptr::null_mut() { return Ok(()); }
|
||||
|
||||
if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, self.index_entries*40) } {
|
||||
if let Err(err) = unsafe {
|
||||
nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, self.index_entries * 40)
|
||||
} {
|
||||
bail!("unmap dynamic index failed - {}", err);
|
||||
}
|
||||
|
||||
|
|
@ -176,7 +180,6 @@ impl DynamicIndexReader {
|
|||
|
||||
/// Compute checksum and data size
|
||||
pub fn compute_csum(&self) -> ([u8; 32], u64) {
|
||||
|
||||
let mut csum = openssl::sha::Sha256::new();
|
||||
let mut chunk_end = 0;
|
||||
for pos in 0..self.index_entries {
|
||||
|
|
@ -213,9 +216,8 @@ impl DynamicIndexReader {
|
|||
start: u64,
|
||||
end_idx: usize,
|
||||
end: u64,
|
||||
offset: u64
|
||||
offset: u64,
|
||||
) -> Result<usize, Error> {
|
||||
|
||||
if (offset >= end) || (offset < start) {
|
||||
bail!("offset out of range");
|
||||
}
|
||||
|
|
@ -243,9 +245,7 @@ impl IndexFile for DynamicIndexReader {
|
|||
if pos >= self.index_entries {
|
||||
None
|
||||
} else {
|
||||
Some(unsafe {
|
||||
std::mem::transmute(self.chunk_digest(pos).as_ptr())
|
||||
})
|
||||
Some(unsafe { std::mem::transmute(self.chunk_digest(pos).as_ptr()) })
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -269,9 +269,7 @@ pub struct BufferedDynamicReader<S> {
|
|||
}
|
||||
|
||||
impl<S: ReadChunk> BufferedDynamicReader<S> {
|
||||
|
||||
pub fn new(index: DynamicIndexReader, store: S) -> Self {
|
||||
|
||||
let archive_size = index.chunk_end(index.index_entries - 1);
|
||||
Self {
|
||||
store,
|
||||
|
|
@ -284,10 +282,11 @@ impl <S: ReadChunk> BufferedDynamicReader<S> {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn archive_size(&self) -> u64 { self.archive_size }
|
||||
pub fn archive_size(&self) -> u64 {
|
||||
self.archive_size
|
||||
}
|
||||
|
||||
fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> {
|
||||
|
||||
let index = &self.index;
|
||||
let (start, end, digest) = index.chunk_info(idx)?;
|
||||
|
||||
|
|
@ -296,7 +295,11 @@ impl <S: ReadChunk> BufferedDynamicReader<S> {
|
|||
let data = self.store.read_chunk(&digest)?;
|
||||
|
||||
if (end - start) != data.len() as u64 {
|
||||
bail!("read chunk with wrong size ({} != {}", (end - start), data.len());
|
||||
bail!(
|
||||
"read chunk with wrong size ({} != {}",
|
||||
(end - start),
|
||||
data.len()
|
||||
);
|
||||
}
|
||||
|
||||
self.read_buffer.clear();
|
||||
|
|
@ -311,18 +314,18 @@ impl <S: ReadChunk> BufferedDynamicReader<S> {
|
|||
}
|
||||
|
||||
impl<S: ReadChunk> crate::tools::BufferedRead for BufferedDynamicReader<S> {
|
||||
|
||||
fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error> {
|
||||
|
||||
if offset == self.archive_size { return Ok(&self.read_buffer[0..0]); }
|
||||
if offset == self.archive_size {
|
||||
return Ok(&self.read_buffer[0..0]);
|
||||
}
|
||||
|
||||
let buffer_len = self.read_buffer.len();
|
||||
let index = &self.index;
|
||||
|
||||
// optimization for sequential read
|
||||
if buffer_len > 0 &&
|
||||
((self.buffered_chunk_idx + 1) < index.index_entries) &&
|
||||
(offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
|
||||
if buffer_len > 0
|
||||
&& ((self.buffered_chunk_idx + 1) < index.index_entries)
|
||||
&& (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
|
||||
{
|
||||
let next_idx = self.buffered_chunk_idx + 1;
|
||||
let next_end = index.chunk_end(next_idx);
|
||||
|
|
@ -333,9 +336,9 @@ impl <S: ReadChunk> crate::tools::BufferedRead for BufferedDynamicReader<S> {
|
|||
}
|
||||
}
|
||||
|
||||
if (buffer_len == 0) ||
|
||||
(offset < self.buffered_chunk_start) ||
|
||||
(offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
|
||||
if (buffer_len == 0)
|
||||
|| (offset < self.buffered_chunk_start)
|
||||
|| (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
|
||||
{
|
||||
let end_idx = index.index_entries - 1;
|
||||
let end = index.chunk_end(end_idx);
|
||||
|
|
@ -349,20 +352,24 @@ impl <S: ReadChunk> crate::tools::BufferedRead for BufferedDynamicReader<S> {
|
|||
}
|
||||
|
||||
impl<S: ReadChunk> std::io::Read for BufferedDynamicReader<S> {
|
||||
|
||||
fn read(&mut self, buf: &mut [u8]) -> Result<usize, std::io::Error> {
|
||||
|
||||
use std::io::{Error, ErrorKind};
|
||||
use crate::tools::BufferedRead;
|
||||
use std::io::{Error, ErrorKind};
|
||||
|
||||
let data = match self.buffered_read(self.read_offset) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return Err(Error::new(ErrorKind::Other, err.to_string())),
|
||||
};
|
||||
|
||||
let n = if data.len() > buf.len() { buf.len() } else { data.len() };
|
||||
let n = if data.len() > buf.len() {
|
||||
buf.len()
|
||||
} else {
|
||||
data.len()
|
||||
};
|
||||
|
||||
unsafe { std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n); }
|
||||
unsafe {
|
||||
std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n);
|
||||
}
|
||||
|
||||
self.read_offset += n as u64;
|
||||
|
||||
|
|
@ -371,9 +378,7 @@ impl <S: ReadChunk> std::io::Read for BufferedDynamicReader<S> {
|
|||
}
|
||||
|
||||
impl<S: ReadChunk> std::io::Seek for BufferedDynamicReader<S> {
|
||||
|
||||
fn seek(&mut self, pos: SeekFrom) -> Result<u64, std::io::Error> {
|
||||
|
||||
let new_offset = match pos {
|
||||
SeekFrom::Start(start_offset) => start_offset as i64,
|
||||
SeekFrom::End(end_offset) => (self.archive_size as i64) + end_offset,
|
||||
|
|
@ -384,7 +389,11 @@ impl <S: ReadChunk> std::io::Seek for BufferedDynamicReader<S> {
|
|||
if (new_offset < 0) || (new_offset > (self.archive_size as i64)) {
|
||||
return Err(Error::new(
|
||||
ErrorKind::Other,
|
||||
format!("seek is out of range {} ([0..{}])", new_offset, self.archive_size)));
|
||||
format!(
|
||||
"seek is out of range {} ([0..{}])",
|
||||
new_offset, self.archive_size
|
||||
),
|
||||
));
|
||||
}
|
||||
self.read_offset = new_offset as u64;
|
||||
|
||||
|
|
@ -406,16 +415,13 @@ pub struct DynamicIndexWriter {
|
|||
}
|
||||
|
||||
impl Drop for DynamicIndexWriter {
|
||||
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors
|
||||
}
|
||||
}
|
||||
|
||||
impl DynamicIndexWriter {
|
||||
|
||||
pub fn create(store: Arc<ChunkStore>, path: &Path) -> Result<Self, Error> {
|
||||
|
||||
let shared_lock = store.try_shared_lock()?;
|
||||
|
||||
let full_path = store.relative_path(path);
|
||||
|
|
@ -423,7 +429,8 @@ impl DynamicIndexWriter {
|
|||
tmp_path.set_extension("tmp_didx");
|
||||
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.create(true).truncate(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(&tmp_path)?;
|
||||
|
|
@ -433,12 +440,15 @@ impl DynamicIndexWriter {
|
|||
let header_size = std::mem::size_of::<DynamicIndexHeader>();
|
||||
|
||||
// todo: use static assertion when available in rust
|
||||
if header_size != 4096 { panic!("got unexpected header size"); }
|
||||
if header_size != 4096 {
|
||||
panic!("got unexpected header size");
|
||||
}
|
||||
|
||||
let ctime = std::time::SystemTime::now().duration_since(
|
||||
std::time::SystemTime::UNIX_EPOCH)?.as_secs();
|
||||
let ctime = std::time::SystemTime::now()
|
||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)?
|
||||
.as_secs();
|
||||
|
||||
let uuid = Uuid::new_v4();
|
||||
let uuid = Uuid::generate();
|
||||
|
||||
let mut buffer = vec::zeroed(header_size);
|
||||
let header = crate::tools::map_struct_mut::<DynamicIndexHeader>(&mut buffer)?;
|
||||
|
|
@ -472,9 +482,11 @@ impl DynamicIndexWriter {
|
|||
}
|
||||
|
||||
pub fn close(&mut self) -> Result<[u8; 32], Error> {
|
||||
|
||||
if self.closed {
|
||||
bail!("cannot close already closed archive index file {:?}", self.filename);
|
||||
bail!(
|
||||
"cannot close already closed archive index file {:?}",
|
||||
self.filename
|
||||
);
|
||||
}
|
||||
|
||||
self.closed = true;
|
||||
|
|
@ -490,7 +502,6 @@ impl DynamicIndexWriter {
|
|||
self.writer.write_all(&index_csum)?;
|
||||
self.writer.flush()?;
|
||||
|
||||
|
||||
if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) {
|
||||
bail!("Atomic rename file {:?} failed - {}", self.filename, err);
|
||||
}
|
||||
|
|
@ -501,7 +512,10 @@ impl DynamicIndexWriter {
|
|||
// fixme: rename to add_digest
|
||||
pub fn add_chunk(&mut self, offset: u64, digest: &[u8; 32]) -> Result<(), Error> {
|
||||
if self.closed {
|
||||
bail!("cannot write to closed dynamic index file {:?}", self.filename);
|
||||
bail!(
|
||||
"cannot write to closed dynamic index file {:?}",
|
||||
self.filename
|
||||
);
|
||||
}
|
||||
|
||||
let offset_le: &[u8; 8] = unsafe { &std::mem::transmute::<u64, [u8; 8]>(offset.to_le()) };
|
||||
|
|
@ -531,7 +545,6 @@ pub struct DynamicChunkWriter {
|
|||
}
|
||||
|
||||
impl DynamicChunkWriter {
|
||||
|
||||
pub fn new(index: DynamicIndexWriter, chunk_size: usize) -> Self {
|
||||
Self {
|
||||
index,
|
||||
|
|
@ -549,7 +562,6 @@ impl DynamicChunkWriter {
|
|||
}
|
||||
|
||||
pub fn close(&mut self) -> Result<(), Error> {
|
||||
|
||||
if self.closed {
|
||||
return Ok(());
|
||||
}
|
||||
|
|
@ -563,16 +575,18 @@ impl DynamicChunkWriter {
|
|||
self.stat.size = self.chunk_offset as u64;
|
||||
|
||||
// add size of index file
|
||||
self.stat.size += (self.stat.chunk_count*40 + std::mem::size_of::<DynamicIndexHeader>()) as u64;
|
||||
self.stat.size +=
|
||||
(self.stat.chunk_count * 40 + std::mem::size_of::<DynamicIndexHeader>()) as u64;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_chunk_buffer(&mut self) -> Result<(), Error> {
|
||||
|
||||
let chunk_size = self.chunk_buffer.len();
|
||||
|
||||
if chunk_size == 0 { return Ok(()); }
|
||||
if chunk_size == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let expected_chunk_size = self.chunk_offset - self.last_chunk;
|
||||
if expected_chunk_size != self.chunk_buffer.len() {
|
||||
|
|
@ -589,7 +603,6 @@ impl DynamicChunkWriter {
|
|||
|
||||
match self.index.insert_chunk(&chunk, &digest) {
|
||||
Ok((is_duplicate, compressed_size)) => {
|
||||
|
||||
self.stat.compressed_size += compressed_size;
|
||||
if is_duplicate {
|
||||
self.stat.duplicate_chunks += 1;
|
||||
|
|
@ -597,8 +610,14 @@ impl DynamicChunkWriter {
|
|||
self.stat.disk_size += compressed_size;
|
||||
}
|
||||
|
||||
println!("ADD CHUNK {:016x} {} {}% {} {}", self.chunk_offset, chunk_size,
|
||||
(compressed_size*100)/(chunk_size as u64), is_duplicate, proxmox::tools::digest_to_hex(&digest));
|
||||
println!(
|
||||
"ADD CHUNK {:016x} {} {}% {} {}",
|
||||
self.chunk_offset,
|
||||
chunk_size,
|
||||
(compressed_size * 100) / (chunk_size as u64),
|
||||
is_duplicate,
|
||||
proxmox::tools::digest_to_hex(&digest)
|
||||
);
|
||||
self.index.add_chunk(self.chunk_offset as u64, &digest)?;
|
||||
self.chunk_buffer.truncate(0);
|
||||
Ok(())
|
||||
|
|
@ -612,9 +631,7 @@ impl DynamicChunkWriter {
|
|||
}
|
||||
|
||||
impl Write for DynamicChunkWriter {
|
||||
|
||||
fn write(&mut self, data: &[u8]) -> std::result::Result<usize, std::io::Error> {
|
||||
|
||||
let chunker = &mut self.chunker;
|
||||
|
||||
let pos = chunker.scan(data);
|
||||
|
|
@ -624,10 +641,12 @@ impl Write for DynamicChunkWriter {
|
|||
self.chunk_offset += pos;
|
||||
|
||||
if let Err(err) = self.write_chunk_buffer() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Other, err.to_string()));
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
err.to_string(),
|
||||
));
|
||||
}
|
||||
Ok(pos)
|
||||
|
||||
} else {
|
||||
self.chunk_offset += data.len();
|
||||
self.chunk_buffer.extend_from_slice(data);
|
||||
|
|
@ -636,6 +655,9 @@ impl Write for DynamicChunkWriter {
|
|||
}
|
||||
|
||||
fn flush(&mut self) -> std::result::Result<(), std::io::Error> {
|
||||
Err(std::io::Error::new(std::io::ErrorKind::Other, "please use close() instead of flush()"))
|
||||
Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"please use close() instead of flush()",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,24 +1,24 @@
|
|||
use failure::*;
|
||||
use std::io::{Seek, SeekFrom};
|
||||
use std::convert::TryInto;
|
||||
use std::io::{Seek, SeekFrom};
|
||||
|
||||
use crate::tools;
|
||||
use super::IndexFile;
|
||||
use super::chunk_stat::*;
|
||||
use super::chunk_store::*;
|
||||
use super::IndexFile;
|
||||
use crate::tools;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::io::Write;
|
||||
use std::fs::File;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use uuid::Uuid;
|
||||
use chrono::{Local, TimeZone};
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::ChunkInfo;
|
||||
use super::read_chunk::*;
|
||||
use super::ChunkInfo;
|
||||
|
||||
use proxmox::tools::io::ReadExt;
|
||||
use proxmox::tools::Uuid;
|
||||
|
||||
/// Header format definition for fixed index files (`.fidx`)
|
||||
#[repr(C)]
|
||||
|
|
@ -52,7 +52,6 @@ unsafe impl Send for FixedIndexReader {}
|
|||
unsafe impl Sync for FixedIndexReader {}
|
||||
|
||||
impl Drop for FixedIndexReader {
|
||||
|
||||
fn drop(&mut self) {
|
||||
if let Err(err) = self.unmap() {
|
||||
eprintln!("Unable to unmap file - {}", err);
|
||||
|
|
@ -61,9 +60,7 @@ impl Drop for FixedIndexReader {
|
|||
}
|
||||
|
||||
impl FixedIndexReader {
|
||||
|
||||
pub fn open(path: &Path) -> Result<Self, Error> {
|
||||
|
||||
File::open(path)
|
||||
.map_err(Error::from)
|
||||
.and_then(|file| Self::new(file))
|
||||
|
|
@ -71,8 +68,9 @@ impl FixedIndexReader {
|
|||
}
|
||||
|
||||
pub fn new(mut file: std::fs::File) -> Result<Self, Error> {
|
||||
|
||||
if let Err(err) = nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock) {
|
||||
if let Err(err) =
|
||||
nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock)
|
||||
{
|
||||
bail!("unable to get shared lock - {}", err);
|
||||
}
|
||||
|
||||
|
|
@ -101,16 +99,23 @@ impl FixedIndexReader {
|
|||
|
||||
let expected_index_size = (stat.st_size as usize) - header_size;
|
||||
if index_size != expected_index_size {
|
||||
bail!("got unexpected file size ({} != {})", index_size, expected_index_size);
|
||||
bail!(
|
||||
"got unexpected file size ({} != {})",
|
||||
index_size,
|
||||
expected_index_size
|
||||
);
|
||||
}
|
||||
|
||||
let data = unsafe { nix::sys::mman::mmap(
|
||||
let data = unsafe {
|
||||
nix::sys::mman::mmap(
|
||||
std::ptr::null_mut(),
|
||||
index_size,
|
||||
nix::sys::mman::ProtFlags::PROT_READ,
|
||||
nix::sys::mman::MapFlags::MAP_PRIVATE,
|
||||
file.as_raw_fd(),
|
||||
header_size as i64) }? as *mut u8;
|
||||
header_size as i64,
|
||||
)
|
||||
}? as *mut u8;
|
||||
|
||||
Ok(Self {
|
||||
_file: file,
|
||||
|
|
@ -125,12 +130,15 @@ impl FixedIndexReader {
|
|||
}
|
||||
|
||||
fn unmap(&mut self) -> Result<(), Error> {
|
||||
|
||||
if self.index == std::ptr::null_mut() { return Ok(()); }
|
||||
if self.index == std::ptr::null_mut() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let index_size = self.index_length * 32;
|
||||
|
||||
if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
|
||||
if let Err(err) =
|
||||
unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) }
|
||||
{
|
||||
bail!("unmap file failed - {}", err);
|
||||
}
|
||||
|
||||
|
|
@ -140,7 +148,6 @@ impl FixedIndexReader {
|
|||
}
|
||||
|
||||
pub fn chunk_info(&self, pos: usize) -> Result<(u64, u64, [u8; 32]), Error> {
|
||||
|
||||
if pos >= self.index_length {
|
||||
bail!("chunk index out of range");
|
||||
}
|
||||
|
|
@ -188,7 +195,6 @@ impl FixedIndexReader {
|
|||
|
||||
/// Compute checksum and data size
|
||||
pub fn compute_csum(&self) -> ([u8; 32], u64) {
|
||||
|
||||
let mut csum = openssl::sha::Sha256::new();
|
||||
let mut chunk_end = 0;
|
||||
for pos in 0..self.index_length {
|
||||
|
|
@ -204,7 +210,10 @@ impl FixedIndexReader {
|
|||
pub fn print_info(&self) {
|
||||
println!("Size: {}", self.size);
|
||||
println!("ChunkSize: {}", self.chunk_size);
|
||||
println!("CTime: {}", Local.timestamp(self.ctime as i64, 0).format("%c"));
|
||||
println!(
|
||||
"CTime: {}",
|
||||
Local.timestamp(self.ctime as i64, 0).format("%c")
|
||||
);
|
||||
println!("UUID: {:?}", self.uuid);
|
||||
}
|
||||
}
|
||||
|
|
@ -245,7 +254,6 @@ pub struct FixedIndexWriter {
|
|||
unsafe impl Send for FixedIndexWriter {}
|
||||
|
||||
impl Drop for FixedIndexWriter {
|
||||
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors
|
||||
if let Err(err) = self.unmap() {
|
||||
|
|
@ -255,10 +263,13 @@ impl Drop for FixedIndexWriter {
|
|||
}
|
||||
|
||||
impl FixedIndexWriter {
|
||||
|
||||
#[allow(clippy::cast_ptr_alignment)]
|
||||
pub fn create(store: Arc<ChunkStore>, path: &Path, size: usize, chunk_size: usize) -> Result<Self, Error> {
|
||||
|
||||
pub fn create(
|
||||
store: Arc<ChunkStore>,
|
||||
path: &Path,
|
||||
size: usize,
|
||||
chunk_size: usize,
|
||||
) -> Result<Self, Error> {
|
||||
let shared_lock = store.try_shared_lock()?;
|
||||
|
||||
let full_path = store.relative_path(path);
|
||||
|
|
@ -266,7 +277,8 @@ impl FixedIndexWriter {
|
|||
tmp_path.set_extension("tmp_fidx");
|
||||
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.create(true).truncate(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(&tmp_path)?;
|
||||
|
|
@ -274,12 +286,15 @@ impl FixedIndexWriter {
|
|||
let header_size = std::mem::size_of::<FixedIndexHeader>();
|
||||
|
||||
// todo: use static assertion when available in rust
|
||||
if header_size != 4096 { panic!("got unexpected header size"); }
|
||||
if header_size != 4096 {
|
||||
panic!("got unexpected header size");
|
||||
}
|
||||
|
||||
let ctime = std::time::SystemTime::now().duration_since(
|
||||
std::time::SystemTime::UNIX_EPOCH)?.as_secs();
|
||||
let ctime = std::time::SystemTime::now()
|
||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)?
|
||||
.as_secs();
|
||||
|
||||
let uuid = Uuid::new_v4();
|
||||
let uuid = Uuid::generate();
|
||||
|
||||
let buffer = vec![0u8; header_size];
|
||||
let header = unsafe { &mut *(buffer.as_ptr() as *mut FixedIndexHeader) };
|
||||
|
|
@ -298,13 +313,16 @@ impl FixedIndexWriter {
|
|||
let index_size = index_length * 32;
|
||||
nix::unistd::ftruncate(file.as_raw_fd(), (header_size + index_size) as i64)?;
|
||||
|
||||
let data = unsafe { nix::sys::mman::mmap(
|
||||
let data = unsafe {
|
||||
nix::sys::mman::mmap(
|
||||
std::ptr::null_mut(),
|
||||
index_size,
|
||||
nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
|
||||
nix::sys::mman::MapFlags::MAP_SHARED,
|
||||
file.as_raw_fd(),
|
||||
header_size as i64) }? as *mut u8;
|
||||
header_size as i64,
|
||||
)
|
||||
}? as *mut u8;
|
||||
|
||||
Ok(Self {
|
||||
store,
|
||||
|
|
@ -326,12 +344,15 @@ impl FixedIndexWriter {
|
|||
}
|
||||
|
||||
fn unmap(&mut self) -> Result<(), Error> {
|
||||
|
||||
if self.index == std::ptr::null_mut() { return Ok(()); }
|
||||
if self.index == std::ptr::null_mut() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let index_size = self.index_length * 32;
|
||||
|
||||
if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
|
||||
if let Err(err) =
|
||||
unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) }
|
||||
{
|
||||
bail!("unmap file {:?} failed - {}", self.tmp_filename, err);
|
||||
}
|
||||
|
||||
|
|
@ -341,8 +362,9 @@ impl FixedIndexWriter {
|
|||
}
|
||||
|
||||
pub fn close(&mut self) -> Result<[u8; 32], Error> {
|
||||
|
||||
if self.index == std::ptr::null_mut() { bail!("cannot close already closed index file."); }
|
||||
if self.index == std::ptr::null_mut() {
|
||||
bail!("cannot close already closed index file.");
|
||||
}
|
||||
|
||||
let index_size = self.index_length * 32;
|
||||
let data = unsafe { std::slice::from_raw_parts(self.index, index_size) };
|
||||
|
|
@ -363,7 +385,6 @@ impl FixedIndexWriter {
|
|||
}
|
||||
|
||||
pub fn check_chunk_alignment(&self, offset: usize, chunk_len: usize) -> Result<usize, Error> {
|
||||
|
||||
if offset < chunk_len {
|
||||
bail!("got chunk with small offset ({} < {}", offset, chunk_len);
|
||||
}
|
||||
|
|
@ -375,9 +396,15 @@ impl FixedIndexWriter {
|
|||
}
|
||||
|
||||
// last chunk can be smaller
|
||||
if ((offset != self.size) && (chunk_len != self.chunk_size)) ||
|
||||
(chunk_len > self.chunk_size) || (chunk_len == 0) {
|
||||
bail!("chunk with unexpected length ({} != {}", chunk_len, self.chunk_size);
|
||||
if ((offset != self.size) && (chunk_len != self.chunk_size))
|
||||
|| (chunk_len > self.chunk_size)
|
||||
|| (chunk_len == 0)
|
||||
{
|
||||
bail!(
|
||||
"chunk with unexpected length ({} != {}",
|
||||
chunk_len,
|
||||
self.chunk_size
|
||||
);
|
||||
}
|
||||
|
||||
if pos & (self.chunk_size - 1) != 0 {
|
||||
|
|
@ -389,22 +416,28 @@ impl FixedIndexWriter {
|
|||
|
||||
// Note: We want to add data out of order, so do not assume any order here.
|
||||
pub fn add_chunk(&mut self, chunk_info: &ChunkInfo, stat: &mut ChunkStat) -> Result<(), Error> {
|
||||
|
||||
let chunk_len = chunk_info.chunk_len as usize;
|
||||
let offset = chunk_info.offset as usize; // end of chunk
|
||||
|
||||
let idx = self.check_chunk_alignment(offset, chunk_len)?;
|
||||
|
||||
let (is_duplicate, compressed_size) =
|
||||
self.store.insert_chunk(&chunk_info.chunk, &chunk_info.digest)?;
|
||||
let (is_duplicate, compressed_size) = self
|
||||
.store
|
||||
.insert_chunk(&chunk_info.chunk, &chunk_info.digest)?;
|
||||
|
||||
stat.chunk_count += 1;
|
||||
stat.compressed_size += compressed_size;
|
||||
|
||||
let digest = &chunk_info.digest;
|
||||
|
||||
println!("ADD CHUNK {} {} {}% {} {}", idx, chunk_len,
|
||||
(compressed_size*100)/(chunk_len as u64), is_duplicate, proxmox::tools::digest_to_hex(digest));
|
||||
println!(
|
||||
"ADD CHUNK {} {} {}% {} {}",
|
||||
idx,
|
||||
chunk_len,
|
||||
(compressed_size * 100) / (chunk_len as u64),
|
||||
is_duplicate,
|
||||
proxmox::tools::digest_to_hex(digest)
|
||||
);
|
||||
|
||||
if is_duplicate {
|
||||
stat.duplicate_chunks += 1;
|
||||
|
|
@ -416,12 +449,17 @@ impl FixedIndexWriter {
|
|||
}
|
||||
|
||||
pub fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> {
|
||||
|
||||
if index >= self.index_length {
|
||||
bail!("add digest failed - index out of range ({} >= {})", index, self.index_length);
|
||||
bail!(
|
||||
"add digest failed - index out of range ({} >= {})",
|
||||
index,
|
||||
self.index_length
|
||||
);
|
||||
}
|
||||
|
||||
if self.index == std::ptr::null_mut() { bail!("cannot write to closed index file."); }
|
||||
if self.index == std::ptr::null_mut() {
|
||||
bail!("cannot write to closed index file.");
|
||||
}
|
||||
|
||||
let index_pos = index * 32;
|
||||
unsafe {
|
||||
|
|
@ -444,9 +482,7 @@ pub struct BufferedFixedReader<S> {
|
|||
}
|
||||
|
||||
impl<S: ReadChunk> BufferedFixedReader<S> {
|
||||
|
||||
pub fn new(index: FixedIndexReader, store: S) -> Self {
|
||||
|
||||
let archive_size = index.size;
|
||||
Self {
|
||||
store,
|
||||
|
|
@ -459,10 +495,11 @@ impl <S: ReadChunk> BufferedFixedReader<S> {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn archive_size(&self) -> u64 { self.archive_size }
|
||||
pub fn archive_size(&self) -> u64 {
|
||||
self.archive_size
|
||||
}
|
||||
|
||||
fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> {
|
||||
|
||||
let index = &self.index;
|
||||
let (start, end, digest) = index.chunk_info(idx)?;
|
||||
|
||||
|
|
@ -471,7 +508,11 @@ impl <S: ReadChunk> BufferedFixedReader<S> {
|
|||
let data = self.store.read_chunk(&digest)?;
|
||||
|
||||
if (end - start) != data.len() as u64 {
|
||||
bail!("read chunk with wrong size ({} != {}", (end - start), data.len());
|
||||
bail!(
|
||||
"read chunk with wrong size ({} != {}",
|
||||
(end - start),
|
||||
data.len()
|
||||
);
|
||||
}
|
||||
|
||||
self.read_buffer.clear();
|
||||
|
|
@ -486,18 +527,18 @@ impl <S: ReadChunk> BufferedFixedReader<S> {
|
|||
}
|
||||
|
||||
impl<S: ReadChunk> crate::tools::BufferedRead for BufferedFixedReader<S> {
|
||||
|
||||
fn buffered_read(&mut self, offset: u64) -> Result<&[u8], Error> {
|
||||
|
||||
if offset == self.archive_size { return Ok(&self.read_buffer[0..0]); }
|
||||
if offset == self.archive_size {
|
||||
return Ok(&self.read_buffer[0..0]);
|
||||
}
|
||||
|
||||
let buffer_len = self.read_buffer.len();
|
||||
let index = &self.index;
|
||||
|
||||
// optimization for sequential read
|
||||
if buffer_len > 0 &&
|
||||
((self.buffered_chunk_idx + 1) < index.index_length) &&
|
||||
(offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
|
||||
if buffer_len > 0
|
||||
&& ((self.buffered_chunk_idx + 1) < index.index_length)
|
||||
&& (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
|
||||
{
|
||||
let next_idx = self.buffered_chunk_idx + 1;
|
||||
let next_end = index.chunk_end(next_idx);
|
||||
|
|
@ -508,9 +549,9 @@ impl <S: ReadChunk> crate::tools::BufferedRead for BufferedFixedReader<S> {
|
|||
}
|
||||
}
|
||||
|
||||
if (buffer_len == 0) ||
|
||||
(offset < self.buffered_chunk_start) ||
|
||||
(offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
|
||||
if (buffer_len == 0)
|
||||
|| (offset < self.buffered_chunk_start)
|
||||
|| (offset >= (self.buffered_chunk_start + (self.read_buffer.len() as u64)))
|
||||
{
|
||||
let idx = (offset / index.chunk_size as u64) as usize;
|
||||
self.buffer_chunk(idx)?;
|
||||
|
|
@ -522,20 +563,24 @@ impl <S: ReadChunk> crate::tools::BufferedRead for BufferedFixedReader<S> {
|
|||
}
|
||||
|
||||
impl<S: ReadChunk> std::io::Read for BufferedFixedReader<S> {
|
||||
|
||||
fn read(&mut self, buf: &mut [u8]) -> Result<usize, std::io::Error> {
|
||||
|
||||
use std::io::{Error, ErrorKind};
|
||||
use crate::tools::BufferedRead;
|
||||
use std::io::{Error, ErrorKind};
|
||||
|
||||
let data = match self.buffered_read(self.read_offset) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return Err(Error::new(ErrorKind::Other, err.to_string())),
|
||||
};
|
||||
|
||||
let n = if data.len() > buf.len() { buf.len() } else { data.len() };
|
||||
let n = if data.len() > buf.len() {
|
||||
buf.len()
|
||||
} else {
|
||||
data.len()
|
||||
};
|
||||
|
||||
unsafe { std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n); }
|
||||
unsafe {
|
||||
std::ptr::copy_nonoverlapping(data.as_ptr(), buf.as_mut_ptr(), n);
|
||||
}
|
||||
|
||||
self.read_offset += n as u64;
|
||||
|
||||
|
|
@ -544,9 +589,7 @@ impl <S: ReadChunk> std::io::Read for BufferedFixedReader<S> {
|
|||
}
|
||||
|
||||
impl<S: ReadChunk> Seek for BufferedFixedReader<S> {
|
||||
|
||||
fn seek(&mut self, pos: SeekFrom) -> Result<u64, std::io::Error> {
|
||||
|
||||
let new_offset = match pos {
|
||||
SeekFrom::Start(start_offset) => start_offset as i64,
|
||||
SeekFrom::End(end_offset) => (self.archive_size as i64) + end_offset,
|
||||
|
|
@ -557,7 +600,11 @@ impl <S: ReadChunk> Seek for BufferedFixedReader<S> {
|
|||
if (new_offset < 0) || (new_offset > (self.archive_size as i64)) {
|
||||
return Err(Error::new(
|
||||
ErrorKind::Other,
|
||||
format!("seek is out of range {} ([0..{}])", new_offset, self.archive_size)));
|
||||
format!(
|
||||
"seek is out of range {} ([0..{}])",
|
||||
new_offset, self.archive_size
|
||||
),
|
||||
));
|
||||
}
|
||||
self.read_offset = new_offset as u64;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue