tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

tape: store datastore name in tape archives and media catalog

Browse Source 
So that we can store multiple datastores on a single media set.
Deduplication is now per datastore (not per media set).
This commit is contained in:
Dietmar Maurer
2021-03-16 12:52:49 +01:00
parent 0e2bf3aa1d
commit 54722acada
9 changed files with 237 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
src/tape/media_catalog.rs
View File

@ -29,6 +29,20 @@ use crate::{
},
};
pub struct DatastoreContent {
pub snapshot_index: HashMap<String, u64>, // snapshot => file_nr
pub chunk_index: HashMap<[u8;32], u64>, // chunk => file_nr
}
impl DatastoreContent {
pub fn new() -> Self {
Self {
chunk_index: HashMap::new(),
snapshot_index: HashMap::new(),
}
}
}
/// The Media Catalog
///
@ -44,13 +58,11 @@ pub struct MediaCatalog {
log_to_stdout: bool,
current_archive: Option<(Uuid, u64)>,
current_archive: Option<(Uuid, u64, String)>, // (uuid, file_nr, store)
last_entry: Option<(Uuid, u64)>,
chunk_index: HashMap<[u8;32], u64>,
snapshot_index: HashMap<String, u64>,
content: HashMap<String, DatastoreContent>,
pending: Vec<u8>,
}
@ -59,8 +71,12 @@ impl MediaCatalog {
/// Magic number for media catalog files.
// openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.0")[0..8]
// Note: this version did not store datastore names (not supported anymore)
pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0: [u8; 8] = [221, 29, 164, 1, 59, 69, 19, 40];
// openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.1")[0..8]
pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1: [u8; 8] = [76, 142, 232, 193, 32, 168, 137, 113];
/// List media with catalogs
pub fn media_with_catalogs(base_path: &Path) -> Result<HashSet<Uuid>, Error> {
let mut catalogs = HashSet::new();
@ -149,15 +165,14 @@ impl MediaCatalog {
log_to_stdout: false,
current_archive: None,
last_entry: None,
chunk_index: HashMap::new(),
snapshot_index: HashMap::new(),
content: HashMap::new(),
pending: Vec::new(),
};
let found_magic_number = me.load_catalog(&mut file)?;
if !found_magic_number {
me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
}
if write {
@ -207,14 +222,13 @@ impl MediaCatalog {
log_to_stdout: false,
current_archive: None,
last_entry: None,
chunk_index: HashMap::new(),
snapshot_index: HashMap::new(),
content: HashMap::new(),
pending: Vec::new(),
};
me.log_to_stdout = log_to_stdout;
me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
me.register_label(&media_id.label.uuid, 0)?;
@ -265,8 +279,8 @@ impl MediaCatalog {
}
/// Accessor to content list
pub fn snapshot_index(&self) -> &HashMap<String, u64> {
&self.snapshot_index
pub fn content(&self) -> &HashMap<String, DatastoreContent> {
&self.content
}
/// Commit pending changes
@ -319,23 +333,35 @@ impl MediaCatalog {
}
/// Test if the catalog already contain a snapshot
pub fn contains_snapshot(&self, snapshot: &str) -> bool {
self.snapshot_index.contains_key(snapshot)
pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
match self.content.get(store) {
None => false,
Some(content) => content.snapshot_index.contains_key(snapshot),
}
}
/// Returns the chunk archive file number
pub fn lookup_snapshot(&self, snapshot: &str) -> Option<u64> {
self.snapshot_index.get(snapshot).copied()
/// Returns the snapshot archive file number
pub fn lookup_snapshot(&self, store: &str, snapshot: &str) -> Option<u64> {
match self.content.get(store) {
None => None,
Some(content) => content.snapshot_index.get(snapshot).copied(),
}
}
/// Test if the catalog already contain a chunk
pub fn contains_chunk(&self, digest: &[u8;32]) -> bool {
self.chunk_index.contains_key(digest)
pub fn contains_chunk(&self, store: &str, digest: &[u8;32]) -> bool {
match self.content.get(store) {
None => false,
Some(content) => content.chunk_index.contains_key(digest),
}
}
/// Returns the chunk archive file number
pub fn lookup_chunk(&self, digest: &[u8;32]) -> Option<u64> {
self.chunk_index.get(digest).copied()
pub fn lookup_chunk(&self, store: &str, digest: &[u8;32]) -> Option<u64> {
match self.content.get(store) {
None => None,
Some(content) => content.chunk_index.get(digest).copied(),
}
}
fn check_register_label(&self, file_number: u64) -> Result<(), Error> {
@ -395,9 +421,9 @@ impl MediaCatalog {
digest: &[u8;32],
) -> Result<(), Error> {
let file_number = match self.current_archive {
let (file_number, store) = match self.current_archive {
None => bail!("register_chunk failed: no archive started"),
Some((_, file_number)) => file_number,
Some((_, file_number, ref store)) => (file_number, store),
};
if self.log_to_stdout {
@ -407,7 +433,12 @@ impl MediaCatalog {
self.pending.push(b'C');
self.pending.extend(digest);
self.chunk_index.insert(*digest, file_number);
match self.content.get_mut(store) {
None => bail!("storage {} not registered - internal error", store),
Some(content) => {
content.chunk_index.insert(*digest, file_number);
}
}
Ok(())
}
@ -440,24 +471,29 @@ impl MediaCatalog {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
) -> Result<(), Error> {
store: &str,
) -> Result<(), Error> {
self.check_start_chunk_archive(file_number)?;
let entry = ChunkArchiveStart {
file_number,
uuid: *uuid.as_bytes(),
store_name_len: u8::try_from(store.len())?,
};
if self.log_to_stdout {
println!("A|{}|{}", file_number, uuid.to_string());
println!("A|{}|{}|{}", file_number, uuid.to_string(), store);
}
self.pending.push(b'A');
unsafe { self.pending.write_le_value(entry)?; }
self.pending.extend(store.as_bytes());
self.current_archive = Some((uuid, file_number));
self.content.entry(store.to_string()).or_insert(DatastoreContent::new());
self.current_archive = Some((uuid, file_number, store.to_string()));
Ok(())
}
@ -466,7 +502,7 @@ impl MediaCatalog {
match self.current_archive {
None => bail!("end_chunk archive failed: not started"),
Some((ref expected_uuid, expected_file_number)) => {
Some((ref expected_uuid, expected_file_number, ..)) => {
if uuid != expected_uuid {
bail!("end_chunk_archive failed: got unexpected uuid");
}
@ -476,7 +512,6 @@ impl MediaCatalog {
}
}
}
Ok(())
}
@ -485,7 +520,7 @@ impl MediaCatalog {
match self.current_archive.take() {
None => bail!("end_chunk_archive failed: not started"),
Some((uuid, file_number)) => {
Some((uuid, file_number, ..)) => {
let entry = ChunkArchiveEnd {
file_number,
@ -539,6 +574,7 @@ impl MediaCatalog {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
store: &str,
snapshot: &str,
) -> Result<(), Error> {
@ -547,19 +583,25 @@ impl MediaCatalog {
let entry = SnapshotEntry {
file_number,
uuid: *uuid.as_bytes(),
store_name_len: u8::try_from(store.len())?,
name_len: u16::try_from(snapshot.len())?,
};
if self.log_to_stdout {
println!("S|{}|{}|{}", file_number, uuid.to_string(), snapshot);
println!("S|{}|{}|{}:{}", file_number, uuid.to_string(), store, snapshot);
}
self.pending.push(b'S');
unsafe { self.pending.write_le_value(entry)?; }
self.pending.extend(store.as_bytes());
self.pending.push(b':');
self.pending.extend(snapshot.as_bytes());
self.snapshot_index.insert(snapshot.to_string(), file_number);
let content = self.content.entry(store.to_string())
.or_insert(DatastoreContent::new());
content.snapshot_index.insert(snapshot.to_string(), file_number);
self.last_entry = Some((uuid, file_number));
@ -581,7 +623,11 @@ impl MediaCatalog {
Ok(true) => { /* OK */ }
Err(err) => bail!("read failed - {}", err),
}
if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
if magic == Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
// only use in unreleased versions
bail!("old catalog format (v1.0) is no longer supported");
}
if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1 {
bail!("wrong magic number");
}
found_magic_number = true;
@ -597,23 +643,35 @@ impl MediaCatalog {
match entry_type[0] {
b'C' => {
let file_number = match self.current_archive {
let (file_number, store) = match self.current_archive {
None => bail!("register_chunk failed: no archive started"),
Some((_, file_number)) => file_number,
Some((_, file_number, ref store)) => (file_number, store),
};
let mut digest = [0u8; 32];
file.read_exact(&mut digest)?;
self.chunk_index.insert(digest, file_number);
match self.content.get_mut(store) {
None => bail!("storage {} not registered - internal error", store),
Some(content) => {
content.chunk_index.insert(digest, file_number);
}
}
}
b'A' => {
let entry: ChunkArchiveStart = unsafe { file.read_le_value()? };
let file_number = entry.file_number;
let uuid = Uuid::from(entry.uuid);
let store_name_len = entry.store_name_len as usize;
let store = file.read_exact_allocated(store_name_len)?;
let store = std::str::from_utf8(&store)?;
self.check_start_chunk_archive(file_number)?;
self.current_archive = Some((uuid, file_number));
}
self.content.entry(store.to_string())
.or_insert(DatastoreContent::new());
self.current_archive = Some((uuid, file_number, store.to_string()));
}
b'E' => {
let entry: ChunkArchiveEnd = unsafe { file.read_le_value()? };
let file_number = entry.file_number;
@ -627,15 +685,22 @@ impl MediaCatalog {
b'S' => {
let entry: SnapshotEntry = unsafe { file.read_le_value()? };
let file_number = entry.file_number;
let store_name_len = entry.store_name_len;
let name_len = entry.name_len;
let uuid = Uuid::from(entry.uuid);
let store = file.read_exact_allocated(store_name_len as usize + 1)?;
let store = std::str::from_utf8(&store[..store_name_len as usize])?;
let snapshot = file.read_exact_allocated(name_len.into())?;
let snapshot = std::str::from_utf8(&snapshot)?;
self.check_register_snapshot(file_number, snapshot)?;
self.snapshot_index.insert(snapshot.to_string(), file_number);
let content = self.content.entry(store.to_string())
.or_insert(DatastoreContent::new());
content.snapshot_index.insert(snapshot.to_string(), file_number);
self.last_entry = Some((uuid, file_number));
}
@ -693,9 +758,9 @@ impl MediaSetCatalog {
}
/// Test if the catalog already contain a snapshot
pub fn contains_snapshot(&self, snapshot: &str) -> bool {
pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
for catalog in self.catalog_list.values() {
if catalog.contains_snapshot(snapshot) {
if catalog.contains_snapshot(store, snapshot) {
return true;
}
}
@ -703,9 +768,9 @@ impl MediaSetCatalog {
}
/// Test if the catalog already contain a chunk
pub fn contains_chunk(&self, digest: &[u8;32]) -> bool {
pub fn contains_chunk(&self, store: &str, digest: &[u8;32]) -> bool {
for catalog in self.catalog_list.values() {
if catalog.contains_chunk(digest) {
if catalog.contains_chunk(store, digest) {
return true;
}
}
@ -727,6 +792,8 @@ struct LabelEntry {
struct ChunkArchiveStart {
file_number: u64,
uuid: [u8;16],
store_name_len: u8,
/* datastore name follows */
}
#[derive(Endian)]
@ -741,6 +808,7 @@ struct ChunkArchiveEnd{
struct SnapshotEntry{
file_number: u64,
uuid: [u8;16],
store_name_len: u8,
name_len: u16,
/* snapshot name follows */
/* datastore name, ':', snapshot name follows */
}