src/backup/archive_index.rs: first try

src/backup.rs

@@ -3,4 +3,5 @@
 pub mod chunker;
 pub mod chunk_store;
 pub mod image_index;
+pub mod archive_index;
 pub mod datastore;

src/backup/archive_index.rs

@@ -0,0 +1,133 @@
+use failure::*;
+
+use super::chunk_store::*;
+use super::chunker::*;
+
+use std::io::{Read, Write};
+use std::fs::File;
+use std::path::{Path, PathBuf};
+use std::os::unix::io::AsRawFd;
+use uuid::Uuid;
+use chrono::{Local, TimeZone};
+
+#[repr(C)]
+pub struct ArchiveIndexHeader {
+    pub magic: [u8; 12],
+    pub version: u32,
+    pub uuid: [u8; 16],
+    pub ctime: u64,
+    reserved: [u8; 4056], // oversall size is one page (4096 bytes)
+}
+
+pub struct ArchiveIndexWriter<'a> {
+    store: &'a ChunkStore,
+    chunker: Chunker,
+    file: File,
+    filename: PathBuf,
+    tmp_filename: PathBuf,
+    uuid: [u8; 16],
+    ctime: u64,
+
+    chunk_offset: usize,
+    last_chunk: usize,
+    chunk_buffer: Vec<u8>,
+}
+
+impl <'a> ArchiveIndexWriter<'a> {
+
+    pub fn create(store: &'a ChunkStore, path: &Path, chunk_size: usize) -> Result<Self, Error> {
+
+        let full_path = store.relative_path(path);
+        let mut tmp_path = full_path.clone();
+        tmp_path.set_extension("tmp_aidx");
+
+        let mut file = std::fs::OpenOptions::new()
+            .create(true).truncate(true)
+            .read(true)
+            .write(true)
+            .open(&tmp_path)?;
+
+        let header_size = std::mem::size_of::<ArchiveIndexHeader>();
+
+        // todo: use static assertion when available in rust
+        if header_size != 4096 { panic!("got unexpected header size"); }
+
+        let ctime = std::time::SystemTime::now().duration_since(
+            std::time::SystemTime::UNIX_EPOCH)?.as_secs();
+
+        let uuid = Uuid::new_v4();
+
+        let mut buffer = vec![0u8; header_size];
+        let header = crate::tools::map_struct_mut::<ArchiveIndexHeader>(&mut buffer)?;
+
+        header.magic = *b"PROXMOX-AIDX";
+        header.version = u32::to_le(1);
+        header.ctime = u64::to_le(ctime);
+        header.uuid = *uuid.as_bytes();
+
+        file.write_all(&buffer)?;
+
+        Ok(Self {
+            store,
+            chunker: Chunker::new(chunk_size),
+            file: file,
+            filename: full_path,
+            tmp_filename: tmp_path,
+            ctime,
+            uuid: *uuid.as_bytes(),
+
+            chunk_offset: 0,
+            last_chunk: 0,
+            chunk_buffer: Vec::with_capacity(chunk_size*4),
+        })
+    }
+}
+
+impl <'a> Write for ArchiveIndexWriter<'a> {
+
+    fn write(&mut self, data: &[u8]) -> std::result::Result<usize, std::io::Error> {
+
+        use std::io::{Error, ErrorKind};
+
+        let chunker = &mut self.chunker;
+
+        let pos = chunker.scan(data);
+
+        if pos > 0 {
+            self.chunk_buffer.extend(&data[0..pos]);
+            self.chunk_offset += pos;
+
+            let chunk_size = self.chunk_buffer.len();
+
+            let expected_chunk_size = self.chunk_offset - self.last_chunk;
+            if expected_chunk_size != self.chunk_buffer.len() {
+                panic!("wrong chunk size {} != {}",
+                       expected_chunk_size, chunk_size);
+            }
+
+            self.last_chunk = self.chunk_offset;
+
+            match self.store.insert_chunk(&self.chunk_buffer) {
+                Ok((is_duplicate, digest)) => {
+                    println!("ADD CHUNK {} {} {} {}", self.chunk_offset, chunk_size, is_duplicate,  digest_to_hex(&digest));
+                    self.chunk_buffer.truncate(0);
+                    return Ok(pos);
+                }
+                Err(err) => {
+                    self.chunk_buffer.truncate(0);
+                    return Err(Error::new(ErrorKind::Other, err.to_string()));
+                }
+            }
+
+        } else {
+            self.chunk_offset += data.len();
+            self.chunk_buffer.extend(data);
+            return Ok(data.len());
+        }
+    }
+
+    fn flush(&mut self) -> std::result::Result<(), std::io::Error> {
+
+        Ok(())
+    }
+}

src/backup/chunker.rs

@@ -205,26 +205,3 @@ impl Chunker {
         self.h ^= BUZHASH_TABLE[(byte as usize)];
     }
 }
-
-impl Write for Chunker {
-
-    fn write(&mut self, data: &[u8]) -> std::result::Result<usize, std::io::Error> {
-
-        let pos = self.scan(data);
-
-        if pos > 0 {
-            self.offset += pos;
-            println!("BOUND {} size {}", self.offset, self.offset - self.last_offset);
-            self.last_offset = self.offset;
-            return Ok(pos);
-        } else {
-            self.offset += data.len();
-            return Ok(data.len());
-        }
-    }
-
-    fn flush(&mut self) -> std::result::Result<(), std::io::Error> {
-
-        Ok(())
-    }
-}

src/backup/datastore.rs

@@ -8,6 +8,7 @@ use std::sync::{Mutex, Arc};
 use crate::config::datastore;
 use super::chunk_store::*;
 use super::image_index::*;
+use super::archive_index::*;
 
 pub struct DataStore {
     chunk_store: ChunkStore,
@@ -76,6 +77,16 @@ impl DataStore {
         Ok(index)
     }
 
+    pub fn create_archive_writer<P: AsRef<Path>>(
+        &self, filename: P,
+        chunk_size: usize
+    ) -> Result<ArchiveIndexWriter, Error> {
+
+        let index = ArchiveIndexWriter::create(&self.chunk_store, filename.as_ref(), chunk_size)?;
+
+        Ok(index)
+    }
+
     pub fn list_images(&self) -> Result<Vec<PathBuf>, Error> {
         let base = self.chunk_store.base_path();
 

src/bin/backup-client.rs

@@ -44,11 +44,11 @@ fn backup_dir(
     //    .truncate(true)
     //    .open("mytest.catar")?;
 
-    let writer = Chunker::new(chunk_size);
+    let mut index = datastore.create_archive_writer(&target, chunk_size)?;
 
     let path = std::path::PathBuf::from(path);
 
-    CaTarEncoder::encode(path, dir, writer)?;
+    CaTarEncoder::encode(path, dir, index)?;
 
     Ok(())
 }

src/catar/encoder.rs

@@ -163,7 +163,7 @@ impl <W: Write> CaTarEncoder<W> {
 
     fn encode_dir(&mut self, dir: &mut nix::dir::Dir)  -> Result<(), Error> {
 
-        println!("encode_dir: {:?} start {}", self.current_path, self.writer_pos);
+        //println!("encode_dir: {:?} start {}", self.current_path, self.writer_pos);
 
         let mut name_list = vec![];
 
@@ -270,7 +270,7 @@ impl <W: Write> CaTarEncoder<W> {
             self.current_path.pop();
         }
 
-        println!("encode_dir: {:?} end {}", self.current_path, self.writer_pos);
+        //println!("encode_dir: {:?} end {}", self.current_path, self.writer_pos);
 
         // fixup goodby item offsets
         let goodbye_start = self.writer_pos as u64;
@@ -282,13 +282,13 @@ impl <W: Write> CaTarEncoder<W> {
 
         self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
 
-        println!("encode_dir: {:?} end1 {}", self.current_path, self.writer_pos);
+        //println!("encode_dir: {:?} end1 {}", self.current_path, self.writer_pos);
         Ok(())
     }
 
     fn encode_file(&mut self, filefd: RawFd)  -> Result<(), Error> {
 
-        println!("encode_file: {:?}", self.current_path);
+        //println!("encode_file: {:?}", self.current_path);
 
         let stat = match nix::sys::stat::fstat(filefd) {
             Ok(stat) => stat,
@@ -338,7 +338,7 @@ impl <W: Write> CaTarEncoder<W> {
 
     fn encode_symlink(&mut self, target: &[u8], stat: &FileStat)  -> Result<(), Error> {
 
-        println!("encode_symlink: {:?} -> {:?}", self.current_path, target);
+        //println!("encode_symlink: {:?} -> {:?}", self.current_path, target);
 
         self.write_entry(stat)?;