tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

avoid chrono dependency, depend on proxmox 0.3.8

Browse Source 
- remove chrono dependency

- depend on proxmox 0.3.8

- remove epoch_now, epoch_now_u64 and epoch_now_f64

- remove tm_editor (moved to proxmox crate)

- use new helpers from proxmox 0.3.8
  * epoch_i64 and epoch_f64
  * parse_rfc3339
  * epoch_to_rfc3339_utc
  * strftime_local

- BackupDir changes:
  * store epoch and rfc3339 string instead of DateTime
  * backup_time_to_string now return a Result
  * remove unnecessary TryFrom<(BackupGroup, i64)> for BackupDir

- DynamicIndexHeader: change ctime to i64

- FixedIndexHeader: change ctime to i64
This commit is contained in:
Dietmar Maurer
2020-09-12 15:10:47 +02:00
parent 58169da46a
commit 6a7be83efe
37 changed files with 198 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
src/backup/backup_info.rs
View File

@ -2,11 +2,8 @@ use crate::tools;
use anyhow::{bail, format_err, Error};
use regex::Regex;
use std::convert::TryFrom;
use std::os::unix::io::RawFd;
use chrono::{DateTime, LocalResult, TimeZone, SecondsFormat, Utc};
use std::path::{PathBuf, Path};
use lazy_static::lazy_static;
@ -106,8 +103,8 @@ impl BackupGroup {
tools::scandir(libc::AT_FDCWD, &path, &BACKUP_DATE_REGEX, |l2_fd, backup_time, file_type| {
if file_type != nix::dir::Type::Directory { return Ok(()); }
let dt = backup_time.parse::<DateTime<Utc>>()?;
let backup_dir = BackupDir::new(self.backup_type.clone(), self.backup_id.clone(), dt.timestamp())?;
let timestamp = proxmox::tools::time::parse_rfc3339(backup_time)?;
let backup_dir = BackupDir::new(self.backup_type.clone(), self.backup_id.clone(), timestamp)?;
let files = list_backup_files(l2_fd, backup_time)?;
list.push(BackupInfo { backup_dir, files });
@ -117,7 +114,7 @@ impl BackupGroup {
Ok(list)
}
pub fn last_successful_backup(&self, base_path: &Path) -> Result<Option<DateTime<Utc>>, Error> {
pub fn last_successful_backup(&self, base_path: &Path) -> Result<Option<i64>, Error> {
let mut last = None;
@ -143,11 +140,11 @@ impl BackupGroup {
}
}
let dt = backup_time.parse::<DateTime<Utc>>()?;
if let Some(last_dt) = last {
if dt > last_dt { last = Some(dt); }
let timestamp = proxmox::tools::time::parse_rfc3339(backup_time)?;
if let Some(last_timestamp) = last {
if timestamp > last_timestamp { last = Some(timestamp); }
} else {
last = Some(dt);
last = Some(timestamp);
}
Ok(())
@ -204,48 +201,51 @@ pub struct BackupDir {
/// Backup group
group: BackupGroup,
/// Backup timestamp
backup_time: DateTime<Utc>,
backup_time: i64,
// backup_time as rfc3339
backup_time_string: String
}
impl BackupDir {
pub fn new<T, U>(backup_type: T, backup_id: U, timestamp: i64) -> Result<Self, Error>
pub fn new<T, U>(backup_type: T, backup_id: U, backup_time: i64) -> Result<Self, Error>
where
T: Into<String>,
U: Into<String>,
{
let group = BackupGroup::new(backup_type.into(), backup_id.into());
BackupDir::new_with_group(group, timestamp)
BackupDir::new_with_group(group, backup_time)
}
pub fn new_with_group(group: BackupGroup, timestamp: i64) -> Result<Self, Error> {
let backup_time = match Utc.timestamp_opt(timestamp, 0) {
LocalResult::Single(time) => time,
_ => bail!("can't create BackupDir with invalid backup time {}", timestamp),
};
Ok(Self { group, backup_time })
pub fn new_with_group(group: BackupGroup, backup_time: i64) -> Result<Self, Error> {
let backup_time_string = Self::backup_time_to_string(backup_time)?;
Ok(Self { group, backup_time, backup_time_string })
}
pub fn group(&self) -> &BackupGroup {
&self.group
}
pub fn backup_time(&self) -> DateTime<Utc> {
pub fn backup_time(&self) -> i64 {
self.backup_time
}
pub fn backup_time_string(&self) -> &str {
&self.backup_time_string
}
pub fn relative_path(&self) -> PathBuf {
let mut relative_path = self.group.group_path();
relative_path.push(Self::backup_time_to_string(self.backup_time));
relative_path.push(self.backup_time_string.clone());
relative_path
}
pub fn backup_time_to_string(backup_time: DateTime<Utc>) -> String {
backup_time.to_rfc3339_opts(SecondsFormat::Secs, true)
pub fn backup_time_to_string(backup_time: i64) -> Result<String, Error> {
// fixme: can this fail? (avoid unwrap)
proxmox::tools::time::epoch_to_rfc3339_utc(backup_time)
}
}
@ -260,8 +260,9 @@ impl std::str::FromStr for BackupDir {
.ok_or_else(|| format_err!("unable to parse backup snapshot path '{}'", path))?;
let group = BackupGroup::new(cap.get(1).unwrap().as_str(), cap.get(2).unwrap().as_str());
let backup_time = cap.get(3).unwrap().as_str().parse::<DateTime<Utc>>()?;
BackupDir::try_from((group, backup_time.timestamp()))
let backup_time_string = cap.get(3).unwrap().as_str().to_owned();
let backup_time = proxmox::tools::time::parse_rfc3339(&backup_time_string)?;
Ok(BackupDir { group, backup_time, backup_time_string })
}
}
@ -269,16 +270,7 @@ impl std::fmt::Display for BackupDir {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let backup_type = self.group.backup_type();
let id = self.group.backup_id();
let time = Self::backup_time_to_string(self.backup_time);
write!(f, "{}/{}/{}", backup_type, id, time)
}
}
impl TryFrom<(BackupGroup, i64)> for BackupDir {
type Error = Error;
fn try_from((group, timestamp): (BackupGroup, i64)) -> Result<Self, Error> {
BackupDir::new_with_group(group, timestamp)
write!(f, "{}/{}/{}", backup_type, id, self.backup_time_string)
}
}
@ -336,13 +328,18 @@ impl BackupInfo {
if file_type != nix::dir::Type::Directory { return Ok(()); }
tools::scandir(l0_fd, backup_type, &BACKUP_ID_REGEX, |l1_fd, backup_id, file_type| {
if file_type != nix::dir::Type::Directory { return Ok(()); }
tools::scandir(l1_fd, backup_id, &BACKUP_DATE_REGEX, |l2_fd, backup_time, file_type| {
tools::scandir(l1_fd, backup_id, &BACKUP_DATE_REGEX, |l2_fd, backup_time_string, file_type| {
if file_type != nix::dir::Type::Directory { return Ok(()); }
let dt = backup_time.parse::<DateTime<Utc>>()?;
let backup_dir = BackupDir::new(backup_type, backup_id, dt.timestamp())?;
let backup_time = proxmox::tools::time::parse_rfc3339(backup_time_string)?;
let files = list_backup_files(l2_fd, backup_time)?;
let backup_dir = BackupDir {
group: BackupGroup::new(backup_type, backup_id),
backup_time,
backup_time_string: backup_time_string.to_owned(),
};
let files = list_backup_files(l2_fd, backup_time_string)?;
list.push(BackupInfo { backup_dir, files });

9
src/backup/catalog.rs
View File

@ -5,7 +5,6 @@ use std::io::{Read, Write, Seek, SeekFrom};
use std::os::unix::ffi::OsStrExt;
use anyhow::{bail, format_err, Error};
use chrono::offset::{TimeZone, Local, LocalResult};
use pathpatterns::{MatchList, MatchType};
use proxmox::tools::io::ReadExt;
@ -533,10 +532,10 @@ impl <R: Read + Seek> CatalogReader<R> {
self.dump_dir(&path, pos)?;
}
CatalogEntryType::File => {
let mtime_string = match Local.timestamp_opt(mtime as i64, 0) {
LocalResult::Single(time) => time.to_rfc3339_opts(chrono::SecondsFormat::Secs, false),
_ => (mtime as i64).to_string(),
};
let mut mtime_string = mtime.to_string();
if let Ok(s) = proxmox::tools::time::strftime_local("%FT%TZ", mtime as i64) {
mtime_string = s;
}
println!(
"{} {:?} {} {}",

5
src/backup/crypt_config.rs
View File

@ -10,7 +10,6 @@
use std::io::Write;
use anyhow::{bail, Error};
use chrono::{Local, DateTime};
use openssl::hash::MessageDigest;
use openssl::pkcs5::pbkdf2_hmac;
use openssl::symm::{decrypt_aead, Cipher, Crypter, Mode};
@ -216,10 +215,10 @@ impl CryptConfig {
pub fn generate_rsa_encoded_key(
&self,
rsa: openssl::rsa::Rsa<openssl::pkey::Public>,
created: DateTime<Local>,
created: i64,
) -> Result<Vec<u8>, Error> {
let modified = Local::now();
let modified = proxmox::tools::time::epoch_i64();
let key_config = super::KeyConfig { kdf: None, created, modified, data: self.enc_key.to_vec() };
let data = serde_json::to_string(&key_config)?.as_bytes().to_vec();

3
src/backup/datastore.rs
View File

@ -6,7 +6,6 @@ use std::convert::TryFrom;
use anyhow::{bail, format_err, Error};
use lazy_static::lazy_static;
use chrono::{DateTime, Utc};
use serde_json::Value;
use proxmox::tools::fs::{replace_file, CreateOptions};
@ -242,7 +241,7 @@ impl DataStore {
/// Returns the time of the last successful backup
///
/// Or None if there is no backup in the group (or the group dir does not exist).
pub fn last_successful_backup(&self, backup_group: &BackupGroup) -> Result<Option<DateTime<Utc>>, Error> {
pub fn last_successful_backup(&self, backup_group: &BackupGroup) -> Result<Option<i64>, Error> {
let base_path = self.base_path();
let mut group_path = base_path.clone();
group_path.push(backup_group.group_path());

14
src/backup/dynamic_index.rs
View File

@ -21,14 +21,14 @@ use super::read_chunk::ReadChunk;
use super::Chunker;
use super::IndexFile;
use super::{DataBlob, DataChunkBuilder};
use crate::tools::{self, epoch_now_u64};
use crate::tools;
/// Header format definition for dynamic index files (`.dixd`)
#[repr(C)]
pub struct DynamicIndexHeader {
pub magic: [u8; 8],
pub uuid: [u8; 16],
pub ctime: u64,
pub ctime: i64,
/// Sha256 over the index ``SHA256(offset1||digest1||offset2||digest2||...)``
pub index_csum: [u8; 32],
reserved: [u8; 4032], // overall size is one page (4096 bytes)
@ -77,7 +77,7 @@ pub struct DynamicIndexReader {
pub size: usize,
index: Mmap<DynamicEntry>,
pub uuid: [u8; 16],
pub ctime: u64,
pub ctime: i64,
pub index_csum: [u8; 32],
}
@ -107,7 +107,7 @@ impl DynamicIndexReader {
bail!("got unknown magic number");
}
let ctime = u64::from_le(header.ctime);
let ctime = proxmox::tools::time::epoch_i64();
let rawfd = file.as_raw_fd();
@ -480,7 +480,7 @@ pub struct DynamicIndexWriter {
tmp_filename: PathBuf,
csum: Option<openssl::sha::Sha256>,
pub uuid: [u8; 16],
pub ctime: u64,
pub ctime: i64,
}
impl Drop for DynamicIndexWriter {
@ -506,13 +506,13 @@ impl DynamicIndexWriter {
let mut writer = BufWriter::with_capacity(1024 * 1024, file);
let ctime = epoch_now_u64()?;
let ctime = proxmox::tools::time::epoch_i64();
let uuid = Uuid::generate();
let mut header = DynamicIndexHeader::zeroed();
header.magic = super::DYNAMIC_SIZED_CHUNK_INDEX_1_0;
header.ctime = u64::to_le(ctime);
header.ctime = i64::to_le(ctime);
header.uuid = *uuid.as_bytes();
// header.index_csum = [0u8; 32];
writer.write_all(header.as_bytes())?;

29
src/backup/fixed_index.rs
View File

@ -4,9 +4,8 @@ use std::io::{Seek, SeekFrom};
use super::chunk_stat::*;
use super::chunk_store::*;
use super::{IndexFile, ChunkReadInfo};
use crate::tools::{self, epoch_now_u64};
use crate::tools;
use chrono::{Local, LocalResult, TimeZone};
use std::fs::File;
use std::io::Write;
use std::os::unix::io::AsRawFd;
@ -23,7 +22,7 @@ use proxmox::tools::Uuid;
pub struct FixedIndexHeader {
pub magic: [u8; 8],
pub uuid: [u8; 16],
pub ctime: u64,
pub ctime: i64,
/// Sha256 over the index ``SHA256(digest1||digest2||...)``
pub index_csum: [u8; 32],
pub size: u64,
@ -41,7 +40,7 @@ pub struct FixedIndexReader {
index_length: usize,
index: *mut u8,
pub uuid: [u8; 16],
pub ctime: u64,
pub ctime: i64,
pub index_csum: [u8; 32],
}
@ -82,7 +81,7 @@ impl FixedIndexReader {
}
let size = u64::from_le(header.size);
let ctime = u64::from_le(header.ctime);
let ctime = i64::from_le(header.ctime);
let chunk_size = u64::from_le(header.chunk_size);
let index_length = ((size + chunk_size - 1) / chunk_size) as usize;
@ -148,13 +147,13 @@ impl FixedIndexReader {
pub fn print_info(&self) {
println!("Size: {}", self.size);
println!("ChunkSize: {}", self.chunk_size);
println!(
"CTime: {}",
match Local.timestamp_opt(self.ctime as i64, 0) {
LocalResult::Single(ctime) => ctime.format("%c").to_string(),
_ => (self.ctime as i64).to_string(),
}
);
let mut ctime_str = self.ctime.to_string();
if let Ok(s) = proxmox::tools::time::strftime_local("%c",self.ctime) {
ctime_str = s;
}
println!("CTime: {}", ctime_str);
println!("UUID: {:?}", self.uuid);
}
}
@ -231,7 +230,7 @@ pub struct FixedIndexWriter {
index_length: usize,
index: *mut u8,
pub uuid: [u8; 16],
pub ctime: u64,
pub ctime: i64,
}
// `index` is mmap()ed which cannot be thread-local so should be sendable
@ -274,7 +273,7 @@ impl FixedIndexWriter {
panic!("got unexpected header size");
}
let ctime = epoch_now_u64()?;
let ctime = proxmox::tools::time::epoch_i64();
let uuid = Uuid::generate();
@ -282,7 +281,7 @@ impl FixedIndexWriter {
let header = unsafe { &mut *(buffer.as_ptr() as *mut FixedIndexHeader) };
header.magic = super::FIXED_SIZED_CHUNK_INDEX_1_0;
header.ctime = u64::to_le(ctime);
header.ctime = i64::to_le(ctime);
header.size = u64::to_le(size as u64);
header.chunk_size = u64::to_le(chunk_size as u64);
header.uuid = *uuid.as_bytes();

17
src/backup/key_derivation.rs
View File

@ -1,7 +1,6 @@
use anyhow::{bail, format_err, Context, Error};
use serde::{Deserialize, Serialize};
use chrono::{Local, DateTime};
use proxmox::tools::fs::{file_get_contents, replace_file, CreateOptions};
use proxmox::try_block;
@ -61,10 +60,10 @@ impl KeyDerivationConfig {
#[derive(Deserialize, Serialize, Debug)]
pub struct KeyConfig {
pub kdf: Option<KeyDerivationConfig>,
#[serde(with = "proxmox::tools::serde::date_time_as_rfc3339")]
pub created: DateTime<Local>,
#[serde(with = "proxmox::tools::serde::date_time_as_rfc3339")]
pub modified: DateTime<Local>,
#[serde(with = "proxmox::tools::serde::epoch_as_rfc3339")]
pub created: i64,
#[serde(with = "proxmox::tools::serde::epoch_as_rfc3339")]
pub modified: i64,
#[serde(with = "proxmox::tools::serde::bytes_as_base64")]
pub data: Vec<u8>,
}
@ -136,7 +135,7 @@ pub fn encrypt_key_with_passphrase(
enc_data.extend_from_slice(&tag);
enc_data.extend_from_slice(&encrypted_key);
let created = Local::now();
let created = proxmox::tools::time::epoch_i64();
Ok(KeyConfig {
kdf: Some(kdf),
@ -149,7 +148,7 @@ pub fn encrypt_key_with_passphrase(
pub fn load_and_decrypt_key(
path: &std::path::Path,
passphrase: &dyn Fn() -> Result<Vec<u8>, Error>,
) -> Result<([u8;32], DateTime<Local>), Error> {
) -> Result<([u8;32], i64), Error> {
do_load_and_decrypt_key(path, passphrase)
.with_context(|| format!("failed to load decryption key from {:?}", path))
}
@ -157,14 +156,14 @@ pub fn load_and_decrypt_key(
fn do_load_and_decrypt_key(
path: &std::path::Path,
passphrase: &dyn Fn() -> Result<Vec<u8>, Error>,
) -> Result<([u8;32], DateTime<Local>), Error> {
) -> Result<([u8;32], i64), Error> {
decrypt_key(&file_get_contents(&path)?, passphrase)
}
pub fn decrypt_key(
mut keydata: &[u8],
passphrase: &dyn Fn() -> Result<Vec<u8>, Error>,
) -> Result<([u8;32], DateTime<Local>), Error> {
) -> Result<([u8;32], i64), Error> {
let key_config: KeyConfig = serde_json::from_reader(&mut keydata)?;
let raw_data = key_config.data;

2
src/backup/manifest.rs
View File

@ -103,7 +103,7 @@ impl BackupManifest {
Self {
backup_type: snapshot.group().backup_type().into(),
backup_id: snapshot.group().backup_id().into(),
backup_time: snapshot.backup_time().timestamp(),
backup_time: snapshot.backup_time(),
files: Vec::new(),
unprotected: json!({}),
signature: None,

61
src/backup/prune.rs
View File

@ -2,18 +2,16 @@ use anyhow::{Error};
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use chrono::{DateTime, Timelike, Datelike, Local};
use super::{BackupDir, BackupInfo};
use super::BackupInfo;
enum PruneMark { Keep, KeepPartial, Remove }
fn mark_selections<F: Fn(DateTime<Local>, &BackupInfo) -> String> (
fn mark_selections<F: Fn(&BackupInfo) -> Result<String, Error>> (
mark: &mut HashMap<PathBuf, PruneMark>,
list: &Vec<BackupInfo>,
keep: usize,
select_id: F,
) {
) -> Result<(), Error> {
let mut include_hash = HashSet::new();
@ -21,8 +19,7 @@ fn mark_selections<F: Fn(DateTime<Local>, &BackupInfo) -> String> (
for info in list {
let backup_id = info.backup_dir.relative_path();
if let Some(PruneMark::Keep) = mark.get(&backup_id) {
let local_time = info.backup_dir.backup_time().with_timezone(&Local);
let sel_id: String = select_id(local_time, &info);
let sel_id: String = select_id(&info)?;
already_included.insert(sel_id);
}
}
@ -30,8 +27,7 @@ fn mark_selections<F: Fn(DateTime<Local>, &BackupInfo) -> String> (
for info in list {
let backup_id = info.backup_dir.relative_path();
if let Some(_) = mark.get(&backup_id) { continue; }
let local_time = info.backup_dir.backup_time().with_timezone(&Local);
let sel_id: String = select_id(local_time, &info);
let sel_id: String = select_id(&info)?;
if already_included.contains(&sel_id) { continue; }
@ -43,6 +39,8 @@ fn mark_selections<F: Fn(DateTime<Local>, &BackupInfo) -> String> (
mark.insert(backup_id, PruneMark::Remove);
}
}
Ok(())
}
fn remove_incomplete_snapshots(
@ -182,44 +180,43 @@ pub fn compute_prune_info(
remove_incomplete_snapshots(&mut mark, &list);
if let Some(keep_last) = options.keep_last {
mark_selections(&mut mark, &list, keep_last as usize, |_local_time, info| {
BackupDir::backup_time_to_string(info.backup_dir.backup_time())
});
mark_selections(&mut mark, &list, keep_last as usize, |info| {
Ok(info.backup_dir.backup_time_string().to_owned())
})?;
}
use proxmox::tools::time::strftime_local;
if let Some(keep_hourly) = options.keep_hourly {
mark_selections(&mut mark, &list, keep_hourly as usize, |local_time, _info| {
format!("{}/{}/{}/{}", local_time.year(), local_time.month(),
local_time.day(), local_time.hour())
});
mark_selections(&mut mark, &list, keep_hourly as usize, |info| {
strftime_local("%Y/%m/%d/%H", info.backup_dir.backup_time())
})?;
}
if let Some(keep_daily) = options.keep_daily {
mark_selections(&mut mark, &list, keep_daily as usize, |local_time, _info| {
format!("{}/{}/{}", local_time.year(), local_time.month(), local_time.day())
});
mark_selections(&mut mark, &list, keep_daily as usize, |info| {
strftime_local("%Y/%m/%d", info.backup_dir.backup_time())
})?;
}
if let Some(keep_weekly) = options.keep_weekly {
mark_selections(&mut mark, &list, keep_weekly as usize, |local_time, _info| {
let iso_week = local_time.iso_week();
let week = iso_week.week();
// Note: This year number might not match the calendar year number.
let iso_week_year = iso_week.year();
format!("{}/{}", iso_week_year, week)
});
mark_selections(&mut mark, &list, keep_weekly as usize, |info| {
// Note: Use iso-week year/week here. This year number
// might not match the calendar year number.
strftime_local("%G/%V", info.backup_dir.backup_time())
})?;
}
if let Some(keep_monthly) = options.keep_monthly {
mark_selections(&mut mark, &list, keep_monthly as usize, |local_time, _info| {
format!("{}/{}", local_time.year(), local_time.month())
});
mark_selections(&mut mark, &list, keep_monthly as usize, |info| {
strftime_local("%Y/%m", info.backup_dir.backup_time())
})?;
}
if let Some(keep_yearly) = options.keep_yearly {
mark_selections(&mut mark, &list, keep_yearly as usize, |local_time, _info| {
format!("{}/{}", local_time.year(), local_time.year())
});
mark_selections(&mut mark, &list, keep_yearly as usize, |info| {
strftime_local("%Y", info.backup_dir.backup_time())
})?;
}
let prune_info: Vec<(BackupInfo, bool)> = list.into_iter()