tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

tape: store datastore name in tape archives and media catalog

Browse Source 
So that we can store multiple datastores on a single media set.
Deduplication is now per datastore (not per media set).
This commit is contained in:
Dietmar Maurer 2021-03-16 12:52:49 +01:00
parent 0e2bf3aa1d
commit 54722acada
9 changed files with 237 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/api2/tape/backup.rs
View File

@ -402,6 +402,8 @@ fn backup_worker(
task_log!(worker, "latest-only: true (only considering latest snapshots)");
}
let datastore_name = datastore.name();
let mut errors = false;
for (group_number, group) in group_list.into_iter().enumerate() {
@ -416,7 +418,7 @@ fn backup_worker(
if latest_only {
progress.group_snapshots = 1;
if let Some(info) = snapshot_list.pop() {
if pool_writer.contains_snapshot(&info.backup_dir.to_string()) {
if pool_writer.contains_snapshot(datastore_name, &info.backup_dir.to_string()) {
task_log!(worker, "skip snapshot {}", info.backup_dir);
continue;
}
@ -433,7 +435,7 @@ fn backup_worker(
} else {
progress.group_snapshots = snapshot_list.len() as u64;
for (snapshot_number, info) in snapshot_list.into_iter().enumerate() {
if pool_writer.contains_snapshot(&info.backup_dir.to_string()) {
if pool_writer.contains_snapshot(datastore_name, &info.backup_dir.to_string()) {
task_log!(worker, "skip snapshot {}", info.backup_dir);
continue;
}
@ -531,7 +533,7 @@ pub fn backup_snapshot(
worker.check_abort()?;
let (leom, _bytes) = pool_writer.append_chunk_archive(worker, &mut chunk_iter)?;
let (leom, _bytes) = pool_writer.append_chunk_archive(worker, &mut chunk_iter, datastore.name())?;
if leom {
pool_writer.set_media_status_full(&uuid)?;

5
src/api2/tape/media.rs
View File

@ -434,7 +434,8 @@ pub fn list_content(
let catalog = MediaCatalog::open(status_path, &media_id.label.uuid, false, false)?;
for snapshot in catalog.snapshot_index().keys() {
for (store, content) in catalog.content() {
for snapshot in content.snapshot_index.keys() {
let backup_dir: BackupDir = snapshot.parse()?;
if let Some(ref backup_type) = filter.backup_type {
@ -453,10 +454,12 @@ pub fn list_content(
media_set_ctime: set.ctime,
seq_nr: set.seq_nr,
snapshot: snapshot.to_owned(),
store: store.to_owned(),
backup_time: backup_dir.backup_time(),
});
}
}
}
Ok(list)
}

37
src/api2/tape/restore.rs
View File

@ -71,11 +71,15 @@ use crate::{
file_formats::{
PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0,
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1,
PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1,
MediaContentHeader,
ChunkArchiveHeader,
ChunkArchiveDecoder,
SnapshotArchiveHeader,
},
drive::{
TapeDriver,
@ -362,10 +366,18 @@ fn restore_archive<'a>(
bail!("unexpected content magic (label)");
}
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0 => {
let snapshot = reader.read_exact_allocated(header.size as usize)?;
let snapshot = std::str::from_utf8(&snapshot)
.map_err(|_| format_err!("found snapshot archive with non-utf8 characters in name"))?;
task_log!(worker, "Found snapshot archive: {} {}", current_file_number, snapshot);
bail!("unexpected snapshot archive version (v1.0)");
}
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1 => {
let header_data = reader.read_exact_allocated(header.size as usize)?;
let archive_header: SnapshotArchiveHeader = serde_json::from_slice(&header_data)
.map_err(|err| format_err!("unable to parse snapshot archive header - {}", err))?;
let datastore_name = archive_header.store;
let snapshot = archive_header.snapshot;
task_log!(worker, "File {}: snapshot archive {}:{}", current_file_number, datastore_name, snapshot);
let backup_dir: BackupDir = snapshot.parse()?;
@ -393,7 +405,7 @@ fn restore_archive<'a>(
task_log!(worker, "skip incomplete snapshot {}", backup_dir);
}
Ok(true) => {
catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, snapshot)?;
catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, &datastore_name, &snapshot)?;
catalog.commit_if_large()?;
}
}
@ -403,17 +415,26 @@ fn restore_archive<'a>(
reader.skip_to_end()?; // read all data
if let Ok(false) = reader.is_incomplete() {
catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, snapshot)?;
catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, &datastore_name, &snapshot)?;
catalog.commit_if_large()?;
}
}
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0 => {
bail!("unexpected chunk archive version (v1.0)");
}
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1 => {
let header_data = reader.read_exact_allocated(header.size as usize)?;
task_log!(worker, "Found chunk archive: {}", current_file_number);
let archive_header: ChunkArchiveHeader = serde_json::from_slice(&header_data)
.map_err(|err| format_err!("unable to parse chunk archive header - {}", err))?;
let source_datastore = archive_header.store;
task_log!(worker, "File {}: chunk archive for datastore '{}'", current_file_number, source_datastore);
let datastore = target.as_ref().map(|t| t.0);
if let Some(chunks) = restore_chunk_archive(worker, reader, datastore, verbose)? {
catalog.start_chunk_archive(Uuid::from(header.uuid), current_file_number)?;
catalog.start_chunk_archive(Uuid::from(header.uuid), current_file_number, &source_datastore)?;
for digest in chunks.iter() {
catalog.register_chunk(&digest)?;
}

2
src/api2/types/tape/media.rs
View File

@ -144,6 +144,8 @@ pub struct MediaContentEntry {
pub seq_nr: u64,
/// Media Pool
pub pool: String,
/// Datastore Name
pub store: String,
/// Backup snapshot
pub snapshot: String,
/// Snapshot creation time (epoch)

18
src/tape/file_formats/chunk_archive.rs
View File

@ -14,9 +14,10 @@ use crate::tape::{
TapeWrite,
file_formats::{
PROXMOX_TAPE_BLOCK_SIZE,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1,
PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
MediaContentHeader,
ChunkArchiveHeader,
ChunkArchiveEntryHeader,
},
};
@ -36,13 +37,20 @@ pub struct ChunkArchiveWriter<'a> {
impl <'a> ChunkArchiveWriter<'a> {
pub const MAGIC: [u8; 8] = PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0;
pub const MAGIC: [u8; 8] = PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1;
/// Creates a new instance
pub fn new(mut writer: Box<dyn TapeWrite + 'a>, close_on_leom: bool) -> Result<(Self,Uuid), Error> {
pub fn new(
mut writer: Box<dyn TapeWrite + 'a>,
store: &str,
close_on_leom: bool,
) -> Result<(Self,Uuid), Error> {
let header = MediaContentHeader::new(Self::MAGIC, 0);
writer.write_header(&header, &[])?;
let archive_header = ChunkArchiveHeader { store: store.to_string() };
let header_data = serde_json::to_string_pretty(&archive_header)?.as_bytes().to_vec();
let header = MediaContentHeader::new(Self::MAGIC, header_data.len() as u32);
writer.write_header(&header, &header_data)?;
let me = Self {
writer: Some(writer),

25
src/tape/file_formats/mod.rs
View File

@ -44,12 +44,19 @@ pub const PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0: [u8; 8] = [42, 5, 191, 60, 176,
pub const PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0: [u8; 8] = [8, 96, 99, 249, 47, 151, 83, 216];
// openssl::sha::sha256(b"Proxmox Backup Chunk Archive v1.0")[0..8]
// only used in unreleased version - no longer supported
pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0: [u8; 8] = [62, 173, 167, 95, 49, 76, 6, 110];
// openssl::sha::sha256(b"Proxmox Backup Chunk Archive v1.1")[0..8]
pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1: [u8; 8] = [109, 49, 99, 109, 215, 2, 131, 191];
// openssl::sha::sha256(b"Proxmox Backup Chunk Archive Entry v1.0")[0..8]
pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0: [u8; 8] = [72, 87, 109, 242, 222, 66, 143, 220];
// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.0")[0..8];
// only used in unreleased version - no longer supported
pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0: [u8; 8] = [9, 182, 2, 31, 125, 232, 114, 133];
// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.1")[0..8];
pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1: [u8; 8] = [218, 22, 21, 208, 17, 226, 154, 98];
lazy_static::lazy_static!{
// Map content magic numbers to human readable names.
@ -58,7 +65,9 @@ lazy_static::lazy_static!{
map.insert(&PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0, "Proxmox Backup Tape Label v1.0");
map.insert(&PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, "Proxmox Backup MediaSet Label v1.0");
map.insert(&PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0, "Proxmox Backup Chunk Archive v1.0");
map.insert(&PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1, "Proxmox Backup Chunk Archive v1.1");
map.insert(&PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, "Proxmox Backup Snapshot Archive v1.0");
map.insert(&PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1, "Proxmox Backup Snapshot Archive v1.1");
map
};
}
@ -172,6 +181,13 @@ impl MediaContentHeader {
}
}
#[derive(Deserialize, Serialize)]
/// Header for chunk archives
pub struct ChunkArchiveHeader {
// Datastore name
pub store: String,
}
#[derive(Endian)]
#[repr(C,packed)]
/// Header for data blobs inside a chunk archive
@ -184,6 +200,15 @@ pub struct ChunkArchiveEntryHeader {
pub size: u64,
}
#[derive(Deserialize, Serialize)]
/// Header for snapshot archives
pub struct SnapshotArchiveHeader {
/// Snapshot name
pub snapshot: String,
/// Datastore name
pub store: String,
}
#[derive(Serialize,Deserialize,Clone,Debug)]
/// Media Label
///

11
src/tape/file_formats/snapshot_archive.rs
View File

@ -12,11 +12,13 @@ use crate::tape::{
SnapshotReader,
file_formats::{
PROXMOX_TAPE_BLOCK_SIZE,
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1,
MediaContentHeader,
SnapshotArchiveHeader,
},
};
/// Write a set of files as `pxar` archive to the tape
///
/// This ignores file attributes like ACLs and xattrs.
@ -31,12 +33,15 @@ pub fn tape_write_snapshot_archive<'a>(
) -> Result<Option<Uuid>, std::io::Error> {
let snapshot = snapshot_reader.snapshot().to_string();
let store = snapshot_reader.datastore_name().to_string();
let file_list = snapshot_reader.file_list();
let header_data = snapshot.as_bytes().to_vec();
let archive_header = SnapshotArchiveHeader { snapshot, store };
let header_data = serde_json::to_string_pretty(&archive_header)?.as_bytes().to_vec();
let header = MediaContentHeader::new(
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, header_data.len() as u32);
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1, header_data.len() as u32);
let content_uuid = header.uuid.into();
let root_metadata = pxar::Metadata::dir_builder(0o0664).build();

152
src/tape/media_catalog.rs
View File

@ -29,6 +29,20 @@ use crate::{
},
};
pub struct DatastoreContent {
pub snapshot_index: HashMap<String, u64>, // snapshot => file_nr
pub chunk_index: HashMap<[u8;32], u64>, // chunk => file_nr
}
impl DatastoreContent {
pub fn new() -> Self {
Self {
chunk_index: HashMap::new(),
snapshot_index: HashMap::new(),
}
}
}
/// The Media Catalog
///
@ -44,13 +58,11 @@ pub struct MediaCatalog {
log_to_stdout: bool,
current_archive: Option<(Uuid, u64)>,
current_archive: Option<(Uuid, u64, String)>, // (uuid, file_nr, store)
last_entry: Option<(Uuid, u64)>,
chunk_index: HashMap<[u8;32], u64>,
snapshot_index: HashMap<String, u64>,
content: HashMap<String, DatastoreContent>,
pending: Vec<u8>,
}
@ -59,8 +71,12 @@ impl MediaCatalog {
/// Magic number for media catalog files.
// openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.0")[0..8]
// Note: this version did not store datastore names (not supported anymore)
pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0: [u8; 8] = [221, 29, 164, 1, 59, 69, 19, 40];
// openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.1")[0..8]
pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1: [u8; 8] = [76, 142, 232, 193, 32, 168, 137, 113];
/// List media with catalogs
pub fn media_with_catalogs(base_path: &Path) -> Result<HashSet<Uuid>, Error> {
let mut catalogs = HashSet::new();
@ -149,15 +165,14 @@ impl MediaCatalog {
log_to_stdout: false,
current_archive: None,
last_entry: None,
chunk_index: HashMap::new(),
snapshot_index: HashMap::new(),
content: HashMap::new(),
pending: Vec::new(),
};
let found_magic_number = me.load_catalog(&mut file)?;
if !found_magic_number {
me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
}
if write {
@ -207,14 +222,13 @@ impl MediaCatalog {
log_to_stdout: false,
current_archive: None,
last_entry: None,
chunk_index: HashMap::new(),
snapshot_index: HashMap::new(),
content: HashMap::new(),
pending: Vec::new(),
};
me.log_to_stdout = log_to_stdout;
me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
me.register_label(&media_id.label.uuid, 0)?;
@ -265,8 +279,8 @@ impl MediaCatalog {
}
/// Accessor to content list
pub fn snapshot_index(&self) -> &HashMap<String, u64> {
&self.snapshot_index
pub fn content(&self) -> &HashMap<String, DatastoreContent> {
&self.content
}
/// Commit pending changes
@ -319,23 +333,35 @@ impl MediaCatalog {
}
/// Test if the catalog already contain a snapshot
pub fn contains_snapshot(&self, snapshot: &str) -> bool {
self.snapshot_index.contains_key(snapshot)
pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
match self.content.get(store) {
None => false,
Some(content) => content.snapshot_index.contains_key(snapshot),
}
}
/// Returns the chunk archive file number
pub fn lookup_snapshot(&self, snapshot: &str) -> Option<u64> {
self.snapshot_index.get(snapshot).copied()
/// Returns the snapshot archive file number
pub fn lookup_snapshot(&self, store: &str, snapshot: &str) -> Option<u64> {
match self.content.get(store) {
None => None,
Some(content) => content.snapshot_index.get(snapshot).copied(),
}
}
/// Test if the catalog already contain a chunk
pub fn contains_chunk(&self, digest: &[u8;32]) -> bool {
self.chunk_index.contains_key(digest)
pub fn contains_chunk(&self, store: &str, digest: &[u8;32]) -> bool {
match self.content.get(store) {
None => false,
Some(content) => content.chunk_index.contains_key(digest),
}
}
/// Returns the chunk archive file number
pub fn lookup_chunk(&self, digest: &[u8;32]) -> Option<u64> {
self.chunk_index.get(digest).copied()
pub fn lookup_chunk(&self, store: &str, digest: &[u8;32]) -> Option<u64> {
match self.content.get(store) {
None => None,
Some(content) => content.chunk_index.get(digest).copied(),
}
}
fn check_register_label(&self, file_number: u64) -> Result<(), Error> {
@ -395,9 +421,9 @@ impl MediaCatalog {
digest: &[u8;32],
) -> Result<(), Error> {
let file_number = match self.current_archive {
let (file_number, store) = match self.current_archive {
None => bail!("register_chunk failed: no archive started"),
Some((_, file_number)) => file_number,
Some((_, file_number, ref store)) => (file_number, store),
};
if self.log_to_stdout {
@ -407,7 +433,12 @@ impl MediaCatalog {
self.pending.push(b'C');
self.pending.extend(digest);
self.chunk_index.insert(*digest, file_number);
match self.content.get_mut(store) {
None => bail!("storage {} not registered - internal error", store),
Some(content) => {
content.chunk_index.insert(*digest, file_number);
}
}
Ok(())
}
@ -440,6 +471,7 @@ impl MediaCatalog {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
store: &str,
) -> Result<(), Error> {
self.check_start_chunk_archive(file_number)?;
@ -447,17 +479,21 @@ impl MediaCatalog {
let entry = ChunkArchiveStart {
file_number,
uuid: *uuid.as_bytes(),
store_name_len: u8::try_from(store.len())?,
};
if self.log_to_stdout {
println!("A|{}|{}", file_number, uuid.to_string());
println!("A|{}|{}|{}", file_number, uuid.to_string(), store);
}
self.pending.push(b'A');
unsafe { self.pending.write_le_value(entry)?; }
self.pending.extend(store.as_bytes());
self.current_archive = Some((uuid, file_number));
self.content.entry(store.to_string()).or_insert(DatastoreContent::new());
self.current_archive = Some((uuid, file_number, store.to_string()));
Ok(())
}
@ -466,7 +502,7 @@ impl MediaCatalog {
match self.current_archive {
None => bail!("end_chunk archive failed: not started"),
Some((ref expected_uuid, expected_file_number)) => {
Some((ref expected_uuid, expected_file_number, ..)) => {
if uuid != expected_uuid {
bail!("end_chunk_archive failed: got unexpected uuid");
}
@ -476,7 +512,6 @@ impl MediaCatalog {
}
}
}
Ok(())
}
@ -485,7 +520,7 @@ impl MediaCatalog {
match self.current_archive.take() {
None => bail!("end_chunk_archive failed: not started"),
Some((uuid, file_number)) => {
Some((uuid, file_number, ..)) => {
let entry = ChunkArchiveEnd {
file_number,
@ -539,6 +574,7 @@ impl MediaCatalog {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
store: &str,
snapshot: &str,
) -> Result<(), Error> {
@ -547,19 +583,25 @@ impl MediaCatalog {
let entry = SnapshotEntry {
file_number,
uuid: *uuid.as_bytes(),
store_name_len: u8::try_from(store.len())?,
name_len: u16::try_from(snapshot.len())?,
};
if self.log_to_stdout {
println!("S|{}|{}|{}", file_number, uuid.to_string(), snapshot);
println!("S|{}|{}|{}:{}", file_number, uuid.to_string(), store, snapshot);
}
self.pending.push(b'S');
unsafe { self.pending.write_le_value(entry)?; }
self.pending.extend(store.as_bytes());
self.pending.push(b':');
self.pending.extend(snapshot.as_bytes());
self.snapshot_index.insert(snapshot.to_string(), file_number);
let content = self.content.entry(store.to_string())
.or_insert(DatastoreContent::new());
content.snapshot_index.insert(snapshot.to_string(), file_number);
self.last_entry = Some((uuid, file_number));
@ -581,7 +623,11 @@ impl MediaCatalog {
Ok(true) => { /* OK */ }
Err(err) => bail!("read failed - {}", err),
}
if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
if magic == Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
// only use in unreleased versions
bail!("old catalog format (v1.0) is no longer supported");
}
if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1 {
bail!("wrong magic number");
}
found_magic_number = true;
@ -597,22 +643,34 @@ impl MediaCatalog {
match entry_type[0] {
b'C' => {
let file_number = match self.current_archive {
let (file_number, store) = match self.current_archive {
None => bail!("register_chunk failed: no archive started"),
Some((_, file_number)) => file_number,
Some((_, file_number, ref store)) => (file_number, store),
};
let mut digest = [0u8; 32];
file.read_exact(&mut digest)?;
self.chunk_index.insert(digest, file_number);
match self.content.get_mut(store) {
None => bail!("storage {} not registered - internal error", store),
Some(content) => {
content.chunk_index.insert(digest, file_number);
}
}
}
b'A' => {
let entry: ChunkArchiveStart = unsafe { file.read_le_value()? };
let file_number = entry.file_number;
let uuid = Uuid::from(entry.uuid);
let store_name_len = entry.store_name_len as usize;
let store = file.read_exact_allocated(store_name_len)?;
let store = std::str::from_utf8(&store)?;
self.check_start_chunk_archive(file_number)?;
self.current_archive = Some((uuid, file_number));
self.content.entry(store.to_string())
.or_insert(DatastoreContent::new());
self.current_archive = Some((uuid, file_number, store.to_string()));
}
b'E' => {
let entry: ChunkArchiveEnd = unsafe { file.read_le_value()? };
@ -627,15 +685,22 @@ impl MediaCatalog {
b'S' => {
let entry: SnapshotEntry = unsafe { file.read_le_value()? };
let file_number = entry.file_number;
let store_name_len = entry.store_name_len;
let name_len = entry.name_len;
let uuid = Uuid::from(entry.uuid);
let store = file.read_exact_allocated(store_name_len as usize + 1)?;
let store = std::str::from_utf8(&store[..store_name_len as usize])?;
let snapshot = file.read_exact_allocated(name_len.into())?;
let snapshot = std::str::from_utf8(&snapshot)?;
self.check_register_snapshot(file_number, snapshot)?;
self.snapshot_index.insert(snapshot.to_string(), file_number);
let content = self.content.entry(store.to_string())
.or_insert(DatastoreContent::new());
content.snapshot_index.insert(snapshot.to_string(), file_number);
self.last_entry = Some((uuid, file_number));
}
@ -693,9 +758,9 @@ impl MediaSetCatalog {
}
/// Test if the catalog already contain a snapshot
pub fn contains_snapshot(&self, snapshot: &str) -> bool {
pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
for catalog in self.catalog_list.values() {
if catalog.contains_snapshot(snapshot) {
if catalog.contains_snapshot(store, snapshot) {
return true;
}
}
@ -703,9 +768,9 @@ impl MediaSetCatalog {
}
/// Test if the catalog already contain a chunk
pub fn contains_chunk(&self, digest: &[u8;32]) -> bool {
pub fn contains_chunk(&self, store: &str, digest: &[u8;32]) -> bool {
for catalog in self.catalog_list.values() {
if catalog.contains_chunk(digest) {
if catalog.contains_chunk(store, digest) {
return true;
}
}
@ -727,6 +792,8 @@ struct LabelEntry {
struct ChunkArchiveStart {
file_number: u64,
uuid: [u8;16],
store_name_len: u8,
/* datastore name follows */
}
#[derive(Endian)]
@ -741,6 +808,7 @@ struct ChunkArchiveEnd{
struct SnapshotEntry{
file_number: u64,
uuid: [u8;16],
store_name_len: u8,
name_len: u16,
/* snapshot name follows */
/* datastore name, ':', snapshot name follows */
}

34
src/tape/pool_writer.rs
View File

@ -50,23 +50,23 @@ pub struct CatalogBuilder {
impl CatalogBuilder {
/// Test if the catalog already contains a snapshot
pub fn contains_snapshot(&self, snapshot: &str) -> bool {
pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
if let Some(ref catalog) = self.catalog {
if catalog.contains_snapshot(snapshot) {
if catalog.contains_snapshot(store, snapshot) {
return true;
}
}
self.media_set_catalog.contains_snapshot(snapshot)
self.media_set_catalog.contains_snapshot(store, snapshot)
}
/// Test if the catalog already contains a chunk
pub fn contains_chunk(&self, digest: &[u8;32]) -> bool {
pub fn contains_chunk(&self, store: &str, digest: &[u8;32]) -> bool {
if let Some(ref catalog) = self.catalog {
if catalog.contains_chunk(digest) {
if catalog.contains_chunk(store, digest) {
return true;
}
}
self.media_set_catalog.contains_chunk(digest)
self.media_set_catalog.contains_chunk(store, digest)
}
/// Add a new catalog, move the old on to the read-only set
@ -90,11 +90,12 @@ impl CatalogBuilder {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
store: &str,
snapshot: &str,
) -> Result<(), Error> {
match self.catalog {
Some(ref mut catalog) => {
catalog.register_snapshot(uuid, file_number, snapshot)?;
catalog.register_snapshot(uuid, file_number, store, snapshot)?;
}
None => bail!("no catalog loaded - internal error"),
}
@ -106,11 +107,12 @@ impl CatalogBuilder {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
store: &str,
chunk_list: &[[u8; 32]],
) -> Result<(), Error> {
match self.catalog {
Some(ref mut catalog) => {
catalog.start_chunk_archive(uuid, file_number)?;
catalog.start_chunk_archive(uuid, file_number, store)?;
for digest in chunk_list {
catalog.register_chunk(digest)?;
}
@ -157,6 +159,8 @@ impl NewChunksIterator {
let mut chunk_index: HashSet<[u8;32]> = HashSet::new();
let datastore_name = snapshot_reader.datastore_name();
let result: Result<(), Error> = proxmox::try_block!({
let mut chunk_iter = snapshot_reader.chunk_iterator()?;
@ -174,7 +178,7 @@ impl NewChunksIterator {
continue;
}
if catalog_builder.lock().unwrap().contains_chunk(&digest) {
if catalog_builder.lock().unwrap().contains_chunk(&datastore_name, &digest) {
continue;
};
@ -279,8 +283,8 @@ impl PoolWriter {
Ok(())
}
pub fn contains_snapshot(&self, snapshot: &str) -> bool {
self.catalog_builder.lock().unwrap().contains_snapshot(snapshot)
pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
self.catalog_builder.lock().unwrap().contains_snapshot(store, snapshot)
}
/// Eject media and drop PoolWriterState (close drive)
@ -462,6 +466,7 @@ impl PoolWriter {
self.catalog_builder.lock().unwrap().register_snapshot(
content_uuid,
current_file_number,
&snapshot_reader.datastore_name().to_string(),
&snapshot_reader.snapshot().to_string(),
)?;
(true, writer.bytes_written())
@ -489,6 +494,7 @@ impl PoolWriter {
&mut self,
worker: &WorkerTask,
chunk_iter: &mut std::iter::Peekable<NewChunksIterator>,
store: &str,
) -> Result<(bool, usize), Error> {
let status = match self.status {
@ -514,6 +520,7 @@ impl PoolWriter {
worker,
writer,
chunk_iter,
store,
MAX_CHUNK_ARCHIVE_SIZE,
)?;
@ -531,7 +538,7 @@ impl PoolWriter {
// register chunks in media_catalog
self.catalog_builder.lock().unwrap()
.register_chunk_archive(content_uuid, current_file_number, &saved_chunks)?;
.register_chunk_archive(content_uuid, current_file_number, store, &saved_chunks)?;
if leom || request_sync {
self.commit()?;
@ -558,10 +565,11 @@ fn write_chunk_archive<'a>(
_worker: &WorkerTask,
writer: Box<dyn 'a + TapeWrite>,
chunk_iter: &mut std::iter::Peekable<NewChunksIterator>,
store: &str,
max_size: usize,
) -> Result<(Vec<[u8;32]>, Uuid, bool, usize), Error> {
let (mut writer, content_uuid) = ChunkArchiveWriter::new(writer, true)?;
let (mut writer, content_uuid) = ChunkArchiveWriter::new(writer, store, true)?;
// we want to get the chunk list in correct order
let mut chunk_list: Vec<[u8;32]> = Vec::new();