From e9c9409a99587a8b58e3905150a7b1099bc651fa Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 8 Mar 2019 16:55:54 +0100 Subject: [PATCH] src/catar/decoder.rs: implement real sequential decoder (no Seek) I copied the old code to src/catar/inspector.rs. Will modify that later to implement a random access decoder ... --- src/bin/catar.rs | 7 +- src/bin/proxmox-backup-client.rs | 19 +- src/catar/decoder.rs | 253 +----------- src/catar/inspector.rs | 656 ++++++++++++++++++++++++++++++ src/client.rs | 3 + src/client/catar_decode_writer.rs | 78 ++++ 6 files changed, 774 insertions(+), 242 deletions(-) create mode 100644 src/catar/inspector.rs create mode 100644 src/client/catar_decode_writer.rs diff --git a/src/bin/catar.rs b/src/bin/catar.rs index 82e95d10..0da88a9d 100644 --- a/src/bin/catar.rs +++ b/src/bin/catar.rs @@ -55,12 +55,14 @@ fn print_filenames( _rpcenv: &mut RpcEnvironment, ) -> Result { + /* FIXME + let archive = tools::required_string_param(¶m, "archive")?; let file = std::fs::File::open(archive)?; let mut reader = std::io::BufReader::new(file); - let mut decoder = CaTarDecoder::new(&mut reader)?; + let mut decoder = CaTarDecoder::new(&mut reader)?; let root = decoder.root(); @@ -68,6 +70,9 @@ fn print_filenames( let mut out = stdout.lock(); decoder.print_filenames(&mut out, &mut PathBuf::from("."), &root)?; + */ + + panic!("not implemented"); Ok(Value::Null) } diff --git a/src/bin/proxmox-backup-client.rs b/src/bin/proxmox-backup-client.rs index 8408a05c..17735370 100644 --- a/src/bin/proxmox-backup-client.rs +++ b/src/bin/proxmox-backup-client.rs @@ -3,7 +3,8 @@ extern crate proxmox_backup; use failure::*; //use std::os::unix::io::AsRawFd; use chrono::{DateTime, Local, TimeZone}; -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::ffi::OsString; use proxmox_backup::tools; use proxmox_backup::cli::*; @@ -427,17 +428,15 @@ fn restore( if file.ends_with(".catar.didx") { let path = format!("api2/json/admin/datastore/{}/catar?{}", repo.store, query); - let mut target = std::path::PathBuf::from(target_path); - target.push(file); - target.set_extension(""); - let fh = std::fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&target)?; + let mut filename = std::path::PathBuf::from(file); + filename.set_extension(""); // remove .didx + filename.set_extension(""); // remove .catar - println!("DOWNLOAD FILE {} to {:?}", path, target); - client.download(&path, Box::new(fh))?; + println!("DOWNLOAD FILE {} to {:?}", path, filename); + let writer = CaTarBackupWriter::new( + &PathBuf::from(target_path), OsString::from(filename), true)?; + client.download(&path, Box::new(writer))?; } else { bail!("unknown file extensions - unable to download '{}'", file); } diff --git a/src/catar/decoder.rs b/src/catar/decoder.rs index bf65b490..875f8871 100644 --- a/src/catar/decoder.rs +++ b/src/catar/decoder.rs @@ -6,15 +6,14 @@ use failure::*; use endian_trait::Endian; use super::format_definition::*; -use crate::tools; -use std::io::{Read, Write, Seek, SeekFrom}; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::os::unix::io::AsRawFd; use std::os::unix::io::RawFd; use std::os::unix::io::FromRawFd; -use std::os::unix::ffi::{OsStrExt, OsStringExt}; +use std::os::unix::ffi::{OsStringExt}; use std::ffi::{OsStr, OsString}; use nix::fcntl::OFlag; @@ -22,49 +21,19 @@ use nix::sys::stat::Mode; use nix::errno::Errno; use nix::NixPath; -pub struct CaDirectoryEntry { - start: u64, - end: u64, - pub filename: OsString, - pub entry: CaFormatEntry, -} - -// This one needs Read+Seek (we may want one without Seek?) -pub struct CaTarDecoder<'a, R: Read + Seek> { +// This one need Read, but works without Seek +pub struct CaTarDecoder<'a, R: Read> { reader: &'a mut R, - root_start: u64, - root_end: u64, + skip_buffer: Vec, } const HEADER_SIZE: u64 = std::mem::size_of::() as u64; -impl <'a, R: Read + Seek> CaTarDecoder<'a, R> { +impl <'a, R: Read> CaTarDecoder<'a, R> { - pub fn new(reader: &'a mut R) -> Result { - - let root_end = reader.seek(SeekFrom::End(0))?; - - Ok(Self { - reader: reader, - root_start: 0, - root_end: root_end, - }) - } - - pub fn root(&self) -> CaDirectoryEntry { - CaDirectoryEntry { - start: self.root_start, - end: self.root_end, - filename: OsString::new(), // Empty - entry: CaFormatEntry { - feature_flags: 0, - mode: 0, - flags: 0, - uid: 0, - gid: 0, - mtime: 0, - } - } + pub fn new(reader: &'a mut R) -> Self { + let mut skip_buffer = vec![0u8; 64*1024]; + Self { reader, skip_buffer } } fn read_item(&mut self) -> Result { @@ -127,33 +96,6 @@ impl <'a, R: Read + Seek> CaTarDecoder<'a, R> { Ok(std::ffi::OsString::from_vec(buffer)) } - pub fn restore Result<(), Error>>( - &mut self, - dir: &CaDirectoryEntry, - callback: F, - ) -> Result<(), Error> { - - let start = dir.start; - - self.reader.seek(SeekFrom::Start(start))?; - - let base = "."; - - let mut path = PathBuf::from(base); - - let dir = match nix::dir::Dir::open(&path, nix::fcntl::OFlag::O_DIRECTORY, nix::sys::stat::Mode::empty()) { - Ok(dir) => dir, - Err(err) => bail!("unable to open base directory - {}", err), - }; - - let restore_dir = "restoretest"; - path.push(restore_dir); - - self.restore_sequential(&mut path, &OsString::from(restore_dir), &dir, &callback)?; - - Ok(()) - } - fn restore_attributes(&mut self, _entry: &CaFormatEntry) -> Result { loop { @@ -267,7 +209,7 @@ impl <'a, R: Read + Seek> CaTarDecoder<'a, R> { pub fn restore_sequential Result<(), Error>>( &mut self, - path: &mut PathBuf, // user for error reporting + path: &mut PathBuf, // used for error reporting filename: &OsStr, // repeats path last component parent: &nix::dir::Dir, callback: &F, @@ -308,7 +250,18 @@ impl <'a, R: Read + Seek> CaTarDecoder<'a, R> { println!("Skip Goodbye"); if head.size < HEADER_SIZE { bail!("detected short goodbye table"); } - self.reader.seek(SeekFrom::Current((head.size - HEADER_SIZE) as i64))?; + + // self.reader.seek(SeekFrom::Current((head.size - HEADER_SIZE) as i64))?; + let mut done = 0; + let skip = (head.size - HEADER_SIZE) as usize; + while done < skip { + let todo = skip - done; + let n = if todo > self.skip_buffer.len() { self.skip_buffer.len() } else { todo }; + let data = &mut self.skip_buffer[..n]; + self.reader.read_exact(data)?; + done += n; + } + self.restore_mode(&entry, dir.as_raw_fd())?; self.restore_mtime(&entry, dir.as_raw_fd())?; @@ -427,168 +380,6 @@ impl <'a, R: Read + Seek> CaTarDecoder<'a, R> { Ok(()) } - - fn read_directory_entry(&mut self, start: u64, end: u64) -> Result { - - self.reader.seek(SeekFrom::Start(start))?; - let mut buffer = [0u8; HEADER_SIZE as usize]; - self.reader.read_exact(&mut buffer)?; - let head = tools::map_struct::(&buffer)?; - - if u64::from_le(head.htype) != CA_FORMAT_FILENAME { - bail!("wrong filename header type for object [{}..{}]", start, end); - } - - let name_len = u64::from_le(head.size); - - let entry_start = start + name_len; - - let filename = self.read_filename(name_len)?; - - let head: CaFormatHeader = self.read_item()?; - check_ca_header::(&head, CA_FORMAT_ENTRY)?; - let entry: CaFormatEntry = self.read_item()?; - - Ok(CaDirectoryEntry { - start: entry_start, - end: end, - filename: filename, - entry: CaFormatEntry { - feature_flags: u64::from_le(entry.feature_flags), - mode: u64::from_le(entry.mode), - flags: u64::from_le(entry.flags), - uid: u64::from_le(entry.uid), - gid: u64::from_le(entry.gid), - mtime: u64::from_le(entry.mtime), - }, - }) - } - - pub fn list_dir(&mut self, dir: &CaDirectoryEntry) -> Result, Error> { - - const GOODBYE_ITEM_SIZE: u64 = std::mem::size_of::() as u64; - - let start = dir.start; - let end = dir.end; - - //println!("list_dir1: {} {}", start, end); - - if (end - start) < (HEADER_SIZE + GOODBYE_ITEM_SIZE) { - bail!("detected short object [{}..{}]", start, end); - } - - self.reader.seek(SeekFrom::Start(end - GOODBYE_ITEM_SIZE))?; - let mut buffer = [0u8; GOODBYE_ITEM_SIZE as usize]; - self.reader.read_exact(&mut buffer)?; - - let item = tools::map_struct::(&buffer)?; - - if u64::from_le(item.hash) != CA_FORMAT_GOODBYE_TAIL_MARKER { - bail!("missing goodbye tail marker for object [{}..{}]", start, end); - } - - let goodbye_table_size = u64::from_le(item.size); - if goodbye_table_size < (HEADER_SIZE + GOODBYE_ITEM_SIZE) { - bail!("short goodbye table size for object [{}..{}]", start, end); - - } - let goodbye_inner_size = goodbye_table_size - HEADER_SIZE - GOODBYE_ITEM_SIZE; - if (goodbye_inner_size % GOODBYE_ITEM_SIZE) != 0 { - bail!("wrong goodbye inner table size for entry [{}..{}]", start, end); - } - - let goodbye_start = end - goodbye_table_size; - - if u64::from_le(item.offset) != (goodbye_start - start) { - println!("DEBUG: {} {}", u64::from_le(item.offset), goodbye_start - start); - bail!("wrong offset in goodbye tail marker for entry [{}..{}]", start, end); - } - - self.reader.seek(SeekFrom::Start(goodbye_start))?; - let mut buffer = [0u8; HEADER_SIZE as usize]; - self.reader.read_exact(&mut buffer)?; - let head = tools::map_struct::(&buffer)?; - - if u64::from_le(head.htype) != CA_FORMAT_GOODBYE { - bail!("wrong goodbye table header type for entry [{}..{}]", start, end); - } - - if u64::from_le(head.size) != goodbye_table_size { - bail!("wrong goodbye table size for entry [{}..{}]", start, end); - } - - let mut buffer = [0u8; GOODBYE_ITEM_SIZE as usize]; - - let mut range_list = Vec::new(); - - for i in 0..goodbye_inner_size/GOODBYE_ITEM_SIZE { - self.reader.read_exact(&mut buffer)?; - let item = tools::map_struct::(&buffer)?; - let item_offset = u64::from_le(item.offset); - if item_offset > (goodbye_start - start) { - bail!("goodbye entry {} offset out of range [{}..{}] {} {} {}", - i, start, end, item_offset, goodbye_start, start); - } - let item_start = goodbye_start - item_offset; - let _item_hash = u64::from_le(item.hash); - let item_end = item_start + u64::from_le(item.size); - if item_end > goodbye_start { - bail!("goodbye entry {} end out of range [{}..{}]", - i, start, end); - } - - range_list.push((item_start, item_end)); - } - - let mut result = vec![]; - - for (item_start, item_end) in range_list { - let entry = self.read_directory_entry(item_start, item_end)?; - //println!("ENTRY: {} {} {:?}", item_start, item_end, entry.filename); - result.push(entry); - } - - Ok(result) - } - - pub fn print_filenames( - &mut self, - output: &mut W, - prefix: &mut PathBuf, - dir: &CaDirectoryEntry, - ) -> Result<(), Error> { - - let mut list = self.list_dir(dir)?; - - list.sort_unstable_by(|a, b| a.filename.cmp(&b.filename)); - - for item in &list { - - prefix.push(item.filename.clone()); - - let mode = item.entry.mode as u32; - - let ifmt = mode & libc::S_IFMT; - - let osstr: &OsStr = prefix.as_ref(); - output.write(osstr.as_bytes())?; - output.write(b"\n")?; - - if ifmt == libc::S_IFDIR { - self.print_filenames(output, prefix, item)?; - } else if ifmt == libc::S_IFREG { - } else if ifmt == libc::S_IFLNK { - } else if ifmt == libc::S_IFBLK { - } else if ifmt == libc::S_IFCHR { - } else { - bail!("unknown item mode/type for {:?}", prefix); - } - - prefix.pop(); - } - - Ok(()) - } } fn file_openat(parent: RawFd, filename: &OsStr, flags: OFlag, mode: Mode) -> Result { diff --git a/src/catar/inspector.rs b/src/catar/inspector.rs new file mode 100644 index 00000000..88835402 --- /dev/null +++ b/src/catar/inspector.rs @@ -0,0 +1,656 @@ +//! *catar* format decoder. +//! +//! This module contain the code to decode *catar* archive files. + +use failure::*; +use endian_trait::Endian; + +use super::format_definition::*; +use crate::tools; + +use std::io::{Read, Write, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +use std::os::unix::io::AsRawFd; +use std::os::unix::io::RawFd; +use std::os::unix::io::FromRawFd; +use std::os::unix::ffi::{OsStrExt, OsStringExt}; +use std::ffi::{OsStr, OsString}; + +use nix::fcntl::OFlag; +use nix::sys::stat::Mode; +use nix::errno::Errno; +use nix::NixPath; + +pub struct CaDirectoryEntry { + start: u64, + end: u64, + pub filename: OsString, + pub entry: CaFormatEntry, +} + +// This one needs Read+Seek (we may want one without Seek?) +pub struct CaTarDecoder<'a, R: Read + Seek> { + reader: &'a mut R, + root_start: u64, + root_end: u64, +} + +const HEADER_SIZE: u64 = std::mem::size_of::() as u64; + +impl <'a, R: Read + Seek> CaTarDecoder<'a, R> { + + pub fn new(reader: &'a mut R) -> Result { + + let root_end = reader.seek(SeekFrom::End(0))?; + + Ok(Self { + reader: reader, + root_start: 0, + root_end: root_end, + }) + } + + pub fn root(&self) -> CaDirectoryEntry { + CaDirectoryEntry { + start: self.root_start, + end: self.root_end, + filename: OsString::new(), // Empty + entry: CaFormatEntry { + feature_flags: 0, + mode: 0, + flags: 0, + uid: 0, + gid: 0, + mtime: 0, + } + } + } + + fn read_item(&mut self) -> Result { + + let mut result: T = unsafe { std::mem::uninitialized() }; + + let buffer = unsafe { std::slice::from_raw_parts_mut( + &mut result as *mut T as *mut u8, + std::mem::size_of::() + )}; + + self.reader.read_exact(buffer)?; + + Ok(result.from_le()) + } + + fn read_symlink(&mut self, size: u64) -> Result { + if size < (HEADER_SIZE + 2) { + bail!("dectected short symlink target."); + } + let target_len = size - HEADER_SIZE; + + if target_len > (libc::PATH_MAX as u64) { + bail!("symlink target too long ({}).", target_len); + } + + let mut buffer = vec![0u8; target_len as usize]; + self.reader.read_exact(&mut buffer)?; + + let last_byte = buffer.pop().unwrap(); + if last_byte != 0u8 { + bail!("symlink target not nul terminated."); + } + + Ok(PathBuf::from(std::ffi::OsString::from_vec(buffer))) + } + + fn read_filename(&mut self, size: u64) -> Result { + if size < (HEADER_SIZE + 2) { + bail!("dectected short filename"); + } + let name_len = size - HEADER_SIZE; + + if name_len > ((libc::FILENAME_MAX as u64) + 1) { + bail!("filename too long ({}).", name_len); + } + + let mut buffer = vec![0u8; name_len as usize]; + self.reader.read_exact(&mut buffer)?; + + let last_byte = buffer.pop().unwrap(); + if last_byte != 0u8 { + bail!("filename entry not nul terminated."); + } + + if buffer.iter().find(|b| (**b == b'/')).is_some() { + bail!("found invalid filename with slashes."); + } + + Ok(std::ffi::OsString::from_vec(buffer)) + } + + pub fn restore Result<(), Error>>( + &mut self, + dir: &CaDirectoryEntry, + callback: F, + ) -> Result<(), Error> { + + let start = dir.start; + + self.reader.seek(SeekFrom::Start(start))?; + + let base = "."; + + let mut path = PathBuf::from(base); + + let dir = match nix::dir::Dir::open(&path, nix::fcntl::OFlag::O_DIRECTORY, nix::sys::stat::Mode::empty()) { + Ok(dir) => dir, + Err(err) => bail!("unable to open base directory - {}", err), + }; + + let restore_dir = "restoretest"; + path.push(restore_dir); + + self.restore_sequential(&mut path, &OsString::from(restore_dir), &dir, &callback)?; + + Ok(()) + } + + fn restore_attributes(&mut self, _entry: &CaFormatEntry) -> Result { + + loop { + let head: CaFormatHeader = self.read_item()?; + match head.htype { + // fimxe: impl ... + _ => return Ok(head), + } + } + } + + fn restore_mode(&mut self, entry: &CaFormatEntry, fd: RawFd) -> Result<(), Error> { + + let mode = Mode::from_bits_truncate((entry.mode as u32) & 0o7777); + + nix::sys::stat::fchmod(fd, mode)?; + + Ok(()) + } + + fn restore_mode_at(&mut self, entry: &CaFormatEntry, dirfd: RawFd, filename: &OsStr) -> Result<(), Error> { + + let mode = Mode::from_bits_truncate((entry.mode as u32) & 0o7777); + + // NOTE: we want :FchmodatFlags::NoFollowSymlink, but fchmodat does not support that + // on linux (see man fchmodat). Fortunately, we can simply avoid calling this on symlinks. + nix::sys::stat::fchmodat(Some(dirfd), filename, mode, nix::sys::stat::FchmodatFlags::FollowSymlink)?; + + Ok(()) + } + + fn restore_ugid(&mut self, entry: &CaFormatEntry, fd: RawFd) -> Result<(), Error> { + + let uid = entry.uid as u32; + let gid = entry.gid as u32; + + let res = unsafe { libc::fchown(fd, uid, gid) }; + Errno::result(res)?; + + Ok(()) + } + + fn restore_ugid_at(&mut self, entry: &CaFormatEntry, dirfd: RawFd, filename: &OsStr) -> Result<(), Error> { + + let uid = entry.uid as u32; + let gid = entry.gid as u32; + + let res = filename.with_nix_path(|cstr| unsafe { + libc::fchownat(dirfd, cstr.as_ptr(), uid, gid, libc::AT_SYMLINK_NOFOLLOW) + })?; + Errno::result(res)?; + + Ok(()) + } + + fn restore_mtime(&mut self, entry: &CaFormatEntry, fd: RawFd) -> Result<(), Error> { + + let times = nsec_to_update_timespec(entry.mtime); + + let res = unsafe { libc::futimens(fd, ×[0]) }; + Errno::result(res)?; + + Ok(()) + } + + fn restore_mtime_at(&mut self, entry: &CaFormatEntry, dirfd: RawFd, filename: &OsStr) -> Result<(), Error> { + + let times = nsec_to_update_timespec(entry.mtime); + + let res = filename.with_nix_path(|cstr| unsafe { + libc::utimensat(dirfd, cstr.as_ptr(), ×[0], libc::AT_SYMLINK_NOFOLLOW) + })?; + Errno::result(res)?; + + Ok(()) + } + + fn restore_device_at(&mut self, entry: &CaFormatEntry, dirfd: RawFd, filename: &OsStr, device: &CaFormatDevice) -> Result<(), Error> { + + let rdev = nix::sys::stat::makedev(device.major, device.minor); + let mode = ((entry.mode as u32) & libc::S_IFMT) | 0o0600; + let res = filename.with_nix_path(|cstr| unsafe { + libc::mknodat(dirfd, cstr.as_ptr(), mode, rdev) + })?; + Errno::result(res)?; + + Ok(()) + } + + fn restore_socket_at(&mut self, dirfd: RawFd, filename: &OsStr) -> Result<(), Error> { + + let mode = libc::S_IFSOCK | 0o0600; + let res = filename.with_nix_path(|cstr| unsafe { + libc::mknodat(dirfd, cstr.as_ptr(), mode, 0) + })?; + Errno::result(res)?; + + Ok(()) + } + + fn restore_fifo_at(&mut self, dirfd: RawFd, filename: &OsStr) -> Result<(), Error> { + + let mode = libc::S_IFIFO | 0o0600; + let res = filename.with_nix_path(|cstr| unsafe { + libc::mkfifoat(dirfd, cstr.as_ptr(), mode) + })?; + Errno::result(res)?; + + Ok(()) + } + + pub fn restore_sequential Result<(), Error>>( + &mut self, + path: &mut PathBuf, // user for error reporting + filename: &OsStr, // repeats path last component + parent: &nix::dir::Dir, + callback: &F, + ) -> Result<(), Error> { + + let parent_fd = parent.as_raw_fd(); + + // read ENTRY first + let head: CaFormatHeader = self.read_item()?; + check_ca_header::(&head, CA_FORMAT_ENTRY)?; + let entry: CaFormatEntry = self.read_item()?; + + let mode = entry.mode as u32; //fixme: upper 32bits? + + let ifmt = mode & libc::S_IFMT; + + if ifmt == libc::S_IFDIR { + let dir = match dir_mkdirat(parent_fd, filename) { + Ok(dir) => dir, + Err(err) => bail!("unable to open directory {:?} - {}", path, err), + }; + + let mut head = self.restore_attributes(&entry)?; + + while head.htype == CA_FORMAT_FILENAME { + let name = self.read_filename(head.size)?; + path.push(&name); + println!("NAME: {:?}", path); + self.restore_sequential(path, &name, &dir, callback)?; + path.pop(); + + head = self.read_item()?; + } + + if head.htype != CA_FORMAT_GOODBYE { + bail!("got unknown header type inside directory entry {:016x}", head.htype); + } + + println!("Skip Goodbye"); + if head.size < HEADER_SIZE { bail!("detected short goodbye table"); } + + // self.reader.seek(SeekFrom::Current((head.size - HEADER_SIZE) as i64))?; + let mut done = 0; + let skip = (head.size - HEADER_SIZE) as usize; + let mut skip_buffer = vec![0u8; 64*1024]; + while done < skip { + let todo = skip - done; + let n = if todo > skip_buffer.len() { skip_buffer.len() } else { todo }; + let data = &mut skip_buffer[..n]; + self.reader.read_exact(data)?; + done += n; + } + + + self.restore_mode(&entry, dir.as_raw_fd())?; + self.restore_mtime(&entry, dir.as_raw_fd())?; + self.restore_ugid(&entry, dir.as_raw_fd())?; + + return Ok(()); + } + + if ifmt == libc::S_IFLNK { + // fixme: create symlink + //fixme: restore permission, acls, xattr, ... + + let head: CaFormatHeader = self.read_item()?; + match head.htype { + CA_FORMAT_SYMLINK => { + let target = self.read_symlink(head.size)?; + println!("TARGET: {:?}", target); + if let Err(err) = symlinkat(&target, parent_fd, filename) { + bail!("create symlink {:?} failed - {}", path, err); + } + } + _ => { + bail!("got unknown header type inside symlink entry {:016x}", head.htype); + } + } + + // self.restore_mode_at(&entry, parent_fd, filename)?; //not supported on symlinks + self.restore_ugid_at(&entry, parent_fd, filename)?; + self.restore_mtime_at(&entry, parent_fd, filename)?; + + return Ok(()); + } + + if ifmt == libc::S_IFSOCK { + + self.restore_socket_at(parent_fd, filename)?; + + self.restore_mode_at(&entry, parent_fd, filename)?; + self.restore_ugid_at(&entry, parent_fd, filename)?; + self.restore_mtime_at(&entry, parent_fd, filename)?; + + return Ok(()); + } + + if ifmt == libc::S_IFIFO { + + self.restore_fifo_at(parent_fd, filename)?; + + self.restore_mode_at(&entry, parent_fd, filename)?; + self.restore_ugid_at(&entry, parent_fd, filename)?; + self.restore_mtime_at(&entry, parent_fd, filename)?; + + return Ok(()); + } + + if (ifmt == libc::S_IFBLK) || (ifmt == libc::S_IFCHR) { + + let head: CaFormatHeader = self.read_item()?; + match head.htype { + CA_FORMAT_DEVICE => { + let device: CaFormatDevice = self.read_item()?; + self.restore_device_at(&entry, parent_fd, filename, &device)?; + } + _ => { + bail!("got unknown header type inside device entry {:016x}", head.htype); + } + } + + self.restore_mode_at(&entry, parent_fd, filename)?; + self.restore_ugid_at(&entry, parent_fd, filename)?; + self.restore_mtime_at(&entry, parent_fd, filename)?; + + return Ok(()); + } + + if ifmt == libc::S_IFREG { + + let mut read_buffer: [u8; 64*1024] = unsafe { std::mem::uninitialized() }; + + let flags = OFlag::O_CREAT|OFlag::O_WRONLY|OFlag::O_EXCL; + let open_mode = Mode::from_bits_truncate(0o0600 | mode); + + let mut file = match file_openat(parent_fd, filename, flags, open_mode) { + Ok(file) => file, + Err(err) => bail!("open file {:?} failed - {}", path, err), + }; + + let head = self.restore_attributes(&entry)?; + + if head.htype != CA_FORMAT_PAYLOAD { + bail!("got unknown header type for file entry {:016x}", head.htype); + } + + if head.size < HEADER_SIZE { + bail!("detected short payload"); + } + let need = (head.size - HEADER_SIZE) as usize; + //self.reader.seek(SeekFrom::Current(need as i64))?; + + let mut done = 0; + while done < need { + let todo = need - done; + let n = if todo > read_buffer.len() { read_buffer.len() } else { todo }; + let data = &mut read_buffer[..n]; + self.reader.read_exact(data)?; + file.write_all(data)?; + done += n; + } + + self.restore_mode(&entry, file.as_raw_fd())?; + self.restore_mtime(&entry, file.as_raw_fd())?; + self.restore_ugid(&entry, file.as_raw_fd())?; + + return Ok(()); + } + + Ok(()) + } + + fn read_directory_entry(&mut self, start: u64, end: u64) -> Result { + + self.reader.seek(SeekFrom::Start(start))?; + let mut buffer = [0u8; HEADER_SIZE as usize]; + self.reader.read_exact(&mut buffer)?; + let head = tools::map_struct::(&buffer)?; + + if u64::from_le(head.htype) != CA_FORMAT_FILENAME { + bail!("wrong filename header type for object [{}..{}]", start, end); + } + + let name_len = u64::from_le(head.size); + + let entry_start = start + name_len; + + let filename = self.read_filename(name_len)?; + + let head: CaFormatHeader = self.read_item()?; + check_ca_header::(&head, CA_FORMAT_ENTRY)?; + let entry: CaFormatEntry = self.read_item()?; + + Ok(CaDirectoryEntry { + start: entry_start, + end: end, + filename: filename, + entry: CaFormatEntry { + feature_flags: u64::from_le(entry.feature_flags), + mode: u64::from_le(entry.mode), + flags: u64::from_le(entry.flags), + uid: u64::from_le(entry.uid), + gid: u64::from_le(entry.gid), + mtime: u64::from_le(entry.mtime), + }, + }) + } + + pub fn list_dir(&mut self, dir: &CaDirectoryEntry) -> Result, Error> { + + const GOODBYE_ITEM_SIZE: u64 = std::mem::size_of::() as u64; + + let start = dir.start; + let end = dir.end; + + //println!("list_dir1: {} {}", start, end); + + if (end - start) < (HEADER_SIZE + GOODBYE_ITEM_SIZE) { + bail!("detected short object [{}..{}]", start, end); + } + + self.reader.seek(SeekFrom::Start(end - GOODBYE_ITEM_SIZE))?; + let mut buffer = [0u8; GOODBYE_ITEM_SIZE as usize]; + self.reader.read_exact(&mut buffer)?; + + let item = tools::map_struct::(&buffer)?; + + if u64::from_le(item.hash) != CA_FORMAT_GOODBYE_TAIL_MARKER { + bail!("missing goodbye tail marker for object [{}..{}]", start, end); + } + + let goodbye_table_size = u64::from_le(item.size); + if goodbye_table_size < (HEADER_SIZE + GOODBYE_ITEM_SIZE) { + bail!("short goodbye table size for object [{}..{}]", start, end); + + } + let goodbye_inner_size = goodbye_table_size - HEADER_SIZE - GOODBYE_ITEM_SIZE; + if (goodbye_inner_size % GOODBYE_ITEM_SIZE) != 0 { + bail!("wrong goodbye inner table size for entry [{}..{}]", start, end); + } + + let goodbye_start = end - goodbye_table_size; + + if u64::from_le(item.offset) != (goodbye_start - start) { + println!("DEBUG: {} {}", u64::from_le(item.offset), goodbye_start - start); + bail!("wrong offset in goodbye tail marker for entry [{}..{}]", start, end); + } + + self.reader.seek(SeekFrom::Start(goodbye_start))?; + let mut buffer = [0u8; HEADER_SIZE as usize]; + self.reader.read_exact(&mut buffer)?; + let head = tools::map_struct::(&buffer)?; + + if u64::from_le(head.htype) != CA_FORMAT_GOODBYE { + bail!("wrong goodbye table header type for entry [{}..{}]", start, end); + } + + if u64::from_le(head.size) != goodbye_table_size { + bail!("wrong goodbye table size for entry [{}..{}]", start, end); + } + + let mut buffer = [0u8; GOODBYE_ITEM_SIZE as usize]; + + let mut range_list = Vec::new(); + + for i in 0..goodbye_inner_size/GOODBYE_ITEM_SIZE { + self.reader.read_exact(&mut buffer)?; + let item = tools::map_struct::(&buffer)?; + let item_offset = u64::from_le(item.offset); + if item_offset > (goodbye_start - start) { + bail!("goodbye entry {} offset out of range [{}..{}] {} {} {}", + i, start, end, item_offset, goodbye_start, start); + } + let item_start = goodbye_start - item_offset; + let _item_hash = u64::from_le(item.hash); + let item_end = item_start + u64::from_le(item.size); + if item_end > goodbye_start { + bail!("goodbye entry {} end out of range [{}..{}]", + i, start, end); + } + + range_list.push((item_start, item_end)); + } + + let mut result = vec![]; + + for (item_start, item_end) in range_list { + let entry = self.read_directory_entry(item_start, item_end)?; + //println!("ENTRY: {} {} {:?}", item_start, item_end, entry.filename); + result.push(entry); + } + + Ok(result) + } + + pub fn print_filenames( + &mut self, + output: &mut W, + prefix: &mut PathBuf, + dir: &CaDirectoryEntry, + ) -> Result<(), Error> { + + let mut list = self.list_dir(dir)?; + + list.sort_unstable_by(|a, b| a.filename.cmp(&b.filename)); + + for item in &list { + + prefix.push(item.filename.clone()); + + let mode = item.entry.mode as u32; + + let ifmt = mode & libc::S_IFMT; + + let osstr: &OsStr = prefix.as_ref(); + output.write(osstr.as_bytes())?; + output.write(b"\n")?; + + if ifmt == libc::S_IFDIR { + self.print_filenames(output, prefix, item)?; + } else if ifmt == libc::S_IFREG { + } else if ifmt == libc::S_IFLNK { + } else if ifmt == libc::S_IFBLK { + } else if ifmt == libc::S_IFCHR { + } else { + bail!("unknown item mode/type for {:?}", prefix); + } + + prefix.pop(); + } + + Ok(()) + } +} + +fn file_openat(parent: RawFd, filename: &OsStr, flags: OFlag, mode: Mode) -> Result { + + let fd = filename.with_nix_path(|cstr| { + nix::fcntl::openat(parent, cstr.as_ref(), flags, mode) + })??; + + let file = unsafe { std::fs::File::from_raw_fd(fd) }; + + Ok(file) +} + +fn dir_mkdirat(parent: RawFd, filename: &OsStr) -> Result { + + // call mkdirat first + let res = filename.with_nix_path(|cstr| unsafe { + libc::mkdirat(parent, cstr.as_ptr(), libc::S_IRWXU) + })?; + Errno::result(res)?; + + let dir = nix::dir::Dir::openat(parent, filename, OFlag::O_DIRECTORY, Mode::empty())?; + + Ok(dir) +} + +fn symlinkat(target: &Path, parent: RawFd, linkname: &OsStr) -> Result<(), Error> { + + target.with_nix_path(|target| { + linkname.with_nix_path(|linkname| { + let res = unsafe { libc::symlinkat(target.as_ptr(), parent, linkname.as_ptr()) }; + Errno::result(res)?; + Ok(()) + })? + })? +} + +fn nsec_to_update_timespec(mtime_nsec: u64) -> [libc::timespec; 2] { + + // restore mtime + const UTIME_OMIT: i64 = ((1 << 30) - 2); + const NANOS_PER_SEC: i64 = 1_000_000_000; + + let sec = (mtime_nsec as i64) / NANOS_PER_SEC; + let nsec = (mtime_nsec as i64) % NANOS_PER_SEC; + + let times: [libc::timespec; 2] = [ + libc::timespec { tv_sec: 0, tv_nsec: UTIME_OMIT }, + libc::timespec { tv_sec: sec, tv_nsec: nsec }, + ]; + + times +} diff --git a/src/client.rs b/src/client.rs index 5eb7e05a..6c36172f 100644 --- a/src/client.rs +++ b/src/client.rs @@ -9,5 +9,8 @@ pub use http_client::*; mod catar_backup_stream; pub use catar_backup_stream::*; +mod catar_decode_writer; +pub use catar_decode_writer::*; + mod backup_repo; pub use backup_repo::*; diff --git a/src/client/catar_decode_writer.rs b/src/client/catar_decode_writer.rs new file mode 100644 index 00000000..eb5d7e6e --- /dev/null +++ b/src/client/catar_decode_writer.rs @@ -0,0 +1,78 @@ +use failure::*; + +use std::thread; +use std::os::unix::io::FromRawFd; +use std::path::{Path, PathBuf}; +use std::io::Write; +use std::ffi::OsString; + +//use nix::fcntl::OFlag; +//use nix::sys::stat::Mode; +//use nix::dir::Dir; + +use crate::catar::decoder::*; + +/// Writer implementation to deccode a .catar archive (download). + +pub struct CaTarBackupWriter { + pipe: Option, + child: Option>, +} + +impl Drop for CaTarBackupWriter { + + fn drop(&mut self) { + drop(self.pipe.take()); + self.child.take().unwrap().join().unwrap(); + } +} + +impl CaTarBackupWriter { + + pub fn new(base: &Path, subdir: OsString, verbose: bool) -> Result { + let (rx, tx) = nix::unistd::pipe()?; + + let dir = match nix::dir::Dir::open(base, nix::fcntl::OFlag::O_DIRECTORY, nix::sys::stat::Mode::empty()) { + Ok(dir) => dir, + Err(err) => bail!("unable to open target directory {:?} - {}", base, err), + }; + let mut path = PathBuf::from(base); + path.push(&subdir); + + let child = thread::spawn(move|| { + let mut reader = unsafe { std::fs::File::from_raw_fd(rx) }; + let mut decoder = CaTarDecoder::new(&mut reader); + + + if let Err(err) = decoder.restore_sequential(&mut path, &subdir, &dir, & |path| { + println!("RESTORE: {:?}", path); + Ok(()) + }) { + eprintln!("catar decode failed - {}", err); + } + }); + + let pipe = unsafe { std::fs::File::from_raw_fd(tx) }; + + Ok(Self { pipe: Some(pipe), child: Some(child) }) + } +} + +impl Write for CaTarBackupWriter { + + fn write(&mut self, buffer: &[u8]) -> Result { + let pipe = match self.pipe { + Some(ref mut pipe) => pipe, + None => unreachable!(), + }; + pipe.write(buffer) + } + + fn flush(&mut self) -> Result<(), std::io::Error> { + let pipe = match self.pipe { + Some(ref mut pipe) => pipe, + None => unreachable!(), + }; + pipe.flush() + } +}