From 4f6892eb74a60893253c8875db80647c35f1a7c1 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 11 Jan 2019 09:20:10 +0100 Subject: [PATCH] catar/encoder.rs: encode linux fs attributes, cleanup encoder Try to avoid duplicate stat calls (assume file tree is stable during backup). --- src/catar/encoder.rs | 182 ++++++++++++++++++++++----------- src/catar/format_definition.rs | 61 ++++++++++- 2 files changed, 180 insertions(+), 63 deletions(-) diff --git a/src/catar/encoder.rs b/src/catar/encoder.rs index e04963d3..ff4617d8 100644 --- a/src/catar/encoder.rs +++ b/src/catar/encoder.rs @@ -3,6 +3,7 @@ //! This module contain the code to generate *catar* archive files. use failure::*; +use endian_trait::Endian; use super::format_definition::*; use super::binary_search_tree::*; @@ -52,7 +53,17 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { }; // todo: use scandirat?? - me.encode_dir(dir)?; + + let stat = match nix::sys::stat::fstat(dir.as_raw_fd()) { + Ok(stat) => stat, + Err(err) => bail!("fstat {:?} failed - {}", me.current_path, err), + }; + + if (stat.st_mode & libc::S_IFMT) != libc::S_IFDIR { + bail!("got unexpected file type {:?} (not a directory)", me.current_path); + } + + me.encode_dir(dir, &stat)?; Ok(()) } @@ -63,6 +74,22 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { Ok(()) } + fn write_item(&mut self, item: &T) -> Result<(), Error> { + + let mut data: T = unsafe { std::mem::uninitialized() }; + + data = (*item).clone().to_le(); + + let buffer = unsafe { std::slice::from_raw_parts( + &data as *const T as *const u8, + std::mem::size_of::() + )}; + + self.write(buffer)?; + + Ok(()) + } + fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> { self.writer.write_all(&self.file_copy_buffer[..size])?; self.writer_pos += size; @@ -71,12 +98,8 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> { - let mut buffer = [0u8; std::mem::size_of::()]; - let mut header = crate::tools::map_struct_mut::(&mut buffer)?; - header.size = u64::to_le((std::mem::size_of::() as u64) + size); - header.htype = u64::to_le(htype); - - self.write(&buffer)?; + let size = size + (std::mem::size_of::() as u64); + self.write_item(&CaFormatHeader { size, htype })?; Ok(()) } @@ -90,33 +113,46 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { Ok(()) } - fn write_entry(&mut self, stat: &FileStat) -> Result<(), Error> { + fn create_entry(&self, stat: &FileStat) -> Result { - let mut buffer = [0u8; std::mem::size_of::() + std::mem::size_of::()]; - let mut header = crate::tools::map_struct_mut::(&mut buffer)?; - header.size = u64::to_le((std::mem::size_of::() + std::mem::size_of::()) as u64); - header.htype = u64::to_le(CA_FORMAT_ENTRY); - - let mut entry = crate::tools::map_struct_mut::(&mut buffer[std::mem::size_of::()..])?; - - entry.feature_flags = u64::to_le(CA_FORMAT_FEATURE_FLAGS_MAX); - - if (stat.st_mode & libc::S_IFMT) == libc::S_IFLNK { - entry.mode = u64::to_le((libc::S_IFLNK | 0o777) as u64); + let mode = if (stat.st_mode & libc::S_IFMT) == libc::S_IFLNK { + (libc::S_IFLNK | 0o777) as u64 } else { - let mode = stat.st_mode & (libc::S_IFMT | 0o7777); - entry.mode = u64::to_le(mode as u64); - } - - entry.flags = 0; // todo: CHATTR, FAT_ATTRS, subvolume? - - entry.uid = u64::to_le(stat.st_uid as u64); - entry.gid = u64::to_le(stat.st_gid as u64); + (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64 + }; let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec; - if mtime > 0 { entry.mtime = mtime as u64 }; + if mtime < 0 { + bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.current_path); + } - self.write(&buffer)?; + + let entry = CaFormatEntry { + feature_flags: CA_FORMAT_FEATURE_FLAGS_MAX, // fixme: ?? + mode: mode, + flags: 0, + uid: stat.st_uid as u64, + gid: stat.st_gid as u64, + mtime: mtime as u64, + }; + + Ok(entry) + } + + fn read_chattr(&self, fd: RawFd, entry: &mut CaFormatEntry) -> Result<(), Error> { + + if let Some(fs_attr) = read_chattr(fd)? { + let flags = ca_feature_flags_from_chattr(fs_attr); + entry.flags = entry.flags | flags; + } + + Ok(()) + } + + fn write_entry(&mut self, entry: &CaFormatEntry) -> Result<(), Error> { + + self.write_header(CA_FORMAT_ENTRY, std::mem::size_of::() as u64)?; + self.write_item(entry)?; Ok(()) } @@ -160,7 +196,7 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { Ok(()) } - fn encode_dir(&mut self, dir: &mut nix::dir::Dir) -> Result<(), Error> { + fn encode_dir(&mut self, dir: &mut nix::dir::Dir, dir_stat: &FileStat) -> Result<(), Error> { //println!("encode_dir: {:?} start {}", self.current_path, self.writer_pos); @@ -168,18 +204,13 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { let rawfd = dir.as_raw_fd(); - let dir_stat = match nix::sys::stat::fstat(rawfd) { - Ok(stat) => stat, - Err(err) => bail!("fstat {:?} failed - {}", self.current_path, err), - }; - - if (dir_stat.st_mode & libc::S_IFMT) != libc::S_IFDIR { - bail!("got unexpected file type {:?} (not a directory)", self.current_path); - } - let dir_start_pos = self.writer_pos; - self.write_entry(&dir_stat)?; + let mut dir_entry = self.create_entry(&dir_stat)?; + + self.read_chattr(rawfd, &mut dir_entry)?; + + self.write_entry(&dir_entry)?; let mut dir_count = 0; @@ -201,22 +232,25 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { if name_len == 2 && name[0] == b'.' && name[1] == 0u8 { continue; } if name_len == 3 && name[0] == b'.' && name[1] == b'.' && name[2] == 0u8 { continue; } - match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) { - Ok(stat) => { - name_list.push((filename, stat)); - } - Err(nix::Error::Sys(Errno::ENOENT)) => self.report_vanished_file(&self.current_path)?, - Err(err) => bail!("fstat {:?} failed - {}", self.current_path, err), - } + name_list.push(filename); } - name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + name_list.sort_unstable_by(|a, b| a.cmp(&b)); let mut goodbye_items = vec![]; - for (filename, stat) in &name_list { + for filename in &name_list { self.current_path.push(std::ffi::OsStr::from_bytes(filename.as_bytes())); + let stat = match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) { + Ok(stat) => stat, + Err(nix::Error::Sys(Errno::ENOENT)) => { + self.report_vanished_file(&self.current_path)?; + continue; + } + Err(err) => bail!("fstat {:?} failed - {}", self.current_path, err), + }; + let start_pos = self.writer_pos; self.write_filename(&filename)?; @@ -224,7 +258,7 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR { match nix::dir::Dir::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) { - Ok(mut dir) => self.encode_dir(&mut dir)?, + Ok(mut dir) => self.encode_dir(&mut dir, &stat)?, Err(nix::Error::Sys(Errno::ENOENT)) => self.report_vanished_file(&self.current_path)?, Err(err) => bail!("open dir {:?} failed - {}", self.current_path, err), } @@ -232,7 +266,7 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { } else if (stat.st_mode & libc::S_IFMT) == libc::S_IFREG { match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) { Ok(filefd) => { - let res = self.encode_file(filefd); + let res = self.encode_file(filefd, &stat); let _ = nix::unistd::close(filefd); // ignore close errors res?; } @@ -285,20 +319,15 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { Ok(()) } - fn encode_file(&mut self, filefd: RawFd) -> Result<(), Error> { + fn encode_file(&mut self, filefd: RawFd, stat: &FileStat) -> Result<(), Error> { //println!("encode_file: {:?}", self.current_path); - let stat = match nix::sys::stat::fstat(filefd) { - Ok(stat) => stat, - Err(err) => bail!("fstat {:?} failed - {}", self.current_path, err), - }; + let mut entry = self.create_entry(&stat)?; - if (stat.st_mode & libc::S_IFMT) != libc::S_IFREG { - bail!("got unexpected file type {:?} (not a regular file)", self.current_path); - } + self.read_chattr(filefd, &mut entry)?; - self.write_entry(&stat)?; + self.write_entry(&entry)?; let size = stat.st_size as u64; @@ -339,7 +368,8 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { //println!("encode_symlink: {:?} -> {:?}", self.current_path, target); - self.write_entry(stat)?; + let entry = self.create_entry(&stat)?; + self.write_entry(&entry)?; self.write_header(CA_FORMAT_SYMLINK, target.len() as u64)?; self.write(target)?; @@ -356,3 +386,33 @@ impl <'a, W: Write> CaTarEncoder<'a, W> { Ok(()) } } + +fn errno_is_unsupported(errno: Errno) -> bool { + + match errno { + Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => { + true + } + _ => false, + } +} + +use nix::{convert_ioctl_res, request_code_read, ioc}; +// /usr/include/linux/fs.h: #define FS_IOC_GETFLAGS _IOR('f', 1, long) +/// read Linux file system attributes (see man chattr) +nix::ioctl_read!(read_attr_fd, b'f', 1, usize); + +fn read_chattr(rawfd: RawFd) -> Result, Error> { + + let mut attr: usize = 0; + + let res = unsafe { read_attr_fd(rawfd, &mut attr)}; + if let Err(err) = res { + if let nix::Error::Sys(errno) = err { + if errno_is_unsupported(errno) { return Ok(None) }; + } + bail!("read_attr_fd failed - {}", err); + } + + Ok(Some(attr as u32)) +} diff --git a/src/catar/format_definition.rs b/src/catar/format_definition.rs index 8c122eae..cb44cf20 100644 --- a/src/catar/format_definition.rs +++ b/src/catar/format_definition.rs @@ -19,9 +19,29 @@ pub const CA_FORMAT_GOODBYE: u64 = 0xdfd35c5e8327c403; /* The end marker used in the GOODBYE object */ pub const CA_FORMAT_GOODBYE_TAIL_MARKER: u64 = 0x57446fa533702943; + +// Feature flags + +// DOS file flags +pub const CA_FORMAT_WITH_FLAG_HIDDEN: u64 = 0x2000; +pub const CA_FORMAT_WITH_FLAG_SYSTEM: u64 = 0x4000; +pub const CA_FORMAT_WITH_FLAG_ARCHIVE: u64 = 0x8000; + +// chattr() flags +pub const CA_FORMAT_WITH_FLAG_APPEND: u64 = 0x10000; +pub const CA_FORMAT_WITH_FLAG_NOATIME: u64 = 0x20000; +pub const CA_FORMAT_WITH_FLAG_COMPR: u64 = 0x40000; +pub const CA_FORMAT_WITH_FLAG_NOCOW: u64 = 0x80000; +pub const CA_FORMAT_WITH_FLAG_NODUMP: u64 = 0x100000; +pub const CA_FORMAT_WITH_FLAG_DIRSYNC: u64 = 0x200000; +pub const CA_FORMAT_WITH_FLAG_IMMUTABLE: u64 = 0x400000; +pub const CA_FORMAT_WITH_FLAG_SYNC: u64 = 0x800000; +pub const CA_FORMAT_WITH_FLAG_NOCOMP: u64 = 0x1000000; +pub const CA_FORMAT_WITH_FLAG_PROJINHERIT: u64 = 0x2000000; + pub const CA_FORMAT_FEATURE_FLAGS_MAX: u64 = 0xb000_0001_ffef_fe26; // fixme: ? -#[derive(Endian)] +#[derive(Endian,Clone)] #[repr(C)] pub struct CaFormatHeader { /// The size of the item, including the size of `CaFormatHeader`. @@ -30,7 +50,7 @@ pub struct CaFormatHeader { pub htype: u64, } -#[derive(Endian)] +#[derive(Endian,Clone)] #[repr(C)] pub struct CaFormatEntry { pub feature_flags: u64, @@ -93,3 +113,40 @@ pub fn check_ca_header(head: &CaFormatHeader, htype: u64) -> Result<(), Error Ok(()) } + +// form /usr/include/linux/fs.h +const FS_APPEND_FL: u32 = 0x00000020; +const FS_NOATIME_FL: u32 = 0x00000080; +const FS_COMPR_FL: u32 = 0x00000004; +const FS_NOCOW_FL: u32 = 0x00800000; +const FS_NODUMP_FL: u32 = 0x00000040; +const FS_DIRSYNC_FL: u32 = 0x00010000; +const FS_IMMUTABLE_FL: u32 = 0x00000010; +const FS_SYNC_FL: u32 = 0x00000008; +const FS_NOCOMP_FL: u32 = 0x00000400; +const FS_PROJINHERIT_FL: u32 = 0x20000000; + +static CHATTR_MAP: [(u64, u32); 10] = [ + ( CA_FORMAT_WITH_FLAG_APPEND, FS_APPEND_FL ), + ( CA_FORMAT_WITH_FLAG_NOATIME, FS_NOATIME_FL ), + ( CA_FORMAT_WITH_FLAG_COMPR, FS_COMPR_FL ), + ( CA_FORMAT_WITH_FLAG_NOCOW, FS_NOCOW_FL ), + ( CA_FORMAT_WITH_FLAG_NODUMP, FS_NODUMP_FL ), + ( CA_FORMAT_WITH_FLAG_DIRSYNC, FS_DIRSYNC_FL ), + ( CA_FORMAT_WITH_FLAG_IMMUTABLE, FS_IMMUTABLE_FL ), + ( CA_FORMAT_WITH_FLAG_SYNC, FS_SYNC_FL ), + ( CA_FORMAT_WITH_FLAG_NOCOMP, FS_NOCOMP_FL ), + ( CA_FORMAT_WITH_FLAG_PROJINHERIT, FS_PROJINHERIT_FL ), +]; + +pub fn ca_feature_flags_from_chattr(attr: u32) -> u64 { + + let mut flags = 0u64; + + for (ca_flag, fs_flag) in &CHATTR_MAP { + if (attr & fs_flag) != 0 { flags = flags | ca_flag; } + } + + + flags +}