tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

split tape code into new pbs_tape workspace

Browse Source
This commit is contained in:
Dietmar Maurer 2021-09-13 11:54:24 +02:00
parent bfd2b47649
commit 048b43af24
48 changed files with 672 additions and 698 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@ -27,6 +27,7 @@ members = [
"pbs-fuse-loop",
"pbs-runtime",
"pbs-systemd",
"pbs-tape",
"pbs-tools",
"proxmox-backup-banner",
@ -108,6 +109,7 @@ pbs-datastore = { path = "pbs-datastore" }
pbs-runtime = { path = "pbs-runtime" }
pbs-systemd = { path = "pbs-systemd" }
pbs-tools = { path = "pbs-tools" }
pbs-tape = { path = "pbs-tape" }
# Local path overrides
# NOTE: You must run `cargo update` after changing this for it to take effect!

5
Makefile
View File

@ -40,6 +40,7 @@ SUBCRATES := \
pbs-fuse-loop \
pbs-runtime \
pbs-systemd \
pbs-tape \
pbs-tools \
proxmox-backup-banner \
proxmox-backup-client \
@ -184,9 +185,11 @@ $(COMPILED_BINS) $(COMPILEDIR)/dump-catalog-shell-cli $(COMPILEDIR)/docgen: .do-
--bin proxmox-file-restore \
--package pxar-bin \
--bin pxar \
--package pbs-tape \
--bin pmt \
--bin pmtx \
--package proxmox-backup \
--bin dump-catalog-shell-cli \
--bin pmt --bin pmtx \
--bin proxmox-daily-update \
--bin proxmox-file-restore \
--bin proxmox-restore-daemon \

25
pbs-tape/Cargo.toml Normal file
View File

@ -0,0 +1,25 @@
[package]
name = "pbs-tape"
version = "0.1.0"
authors = ["Proxmox Support Team <support@proxmox.com>"]
edition = "2018"
description = "LTO tage support"
[dependencies]
lazy_static = "1.4"
libc = "0.2"
anyhow = "1.0"
thiserror = "1.0"
endian_trait = { version = "0.6", features = ["arrays"] }
nix = "0.19.1"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
bitflags = "1.2.1"
regex = "1.2"
udev = ">= 0.3, <0.5"
proxmox = { version = "0.13.0", default-features = false, features = [] }
pbs-api-types = { path = "../pbs-api-types" }
pbs-tools = { path = "../pbs-tools" }
pbs-config = { path = "../pbs-config" }

53
src/bin/pmt.rs → pbs-tape/src/bin/pmt.rs
View File

@ -13,6 +13,8 @@
/// - support volume statistics
/// - read cartridge memory
use std::convert::TryInto;
use anyhow::{bail, Error};
use serde_json::Value;
@ -34,17 +36,9 @@ use pbs_api_types::{
LTO_DRIVE_PATH_SCHEMA, DRIVE_NAME_SCHEMA, LtoTapeDrive,
};
use pbs_config::drive::complete_drive_name;
use proxmox_backup::{
tape::{
complete_drive_path,
lto_tape_device_list,
drive::{
TapeDriver,
LtoTapeHandle,
open_lto_tape_device,
},
},
use pbs_tape::{
sg_tape::SgTape,
linux_list_drives::{complete_drive_path, lto_tape_device_list, open_lto_tape_device},
};
pub const FILE_MARK_COUNT_SCHEMA: Schema =
@ -74,30 +68,30 @@ pub const DRIVE_OPTION_LIST_SCHEMA: Schema =
.min_length(1)
.schema();
fn get_tape_handle(param: &Value) -> Result<LtoTapeHandle, Error> {
fn get_tape_handle(param: &Value) -> Result<SgTape, Error> {
if let Some(name) = param["drive"].as_str() {
let (config, _digest) = pbs_config::drive::config()?;
let drive: LtoTapeDrive = config.lookup("lto", &name)?;
eprintln!("using device {}", drive.path);
return LtoTapeHandle::new(open_lto_tape_device(&drive.path)?);
return SgTape::new(open_lto_tape_device(&drive.path)?);
}
if let Some(device) = param["device"].as_str() {
eprintln!("using device {}", device);
return LtoTapeHandle::new(open_lto_tape_device(&device)?);
return SgTape::new(open_lto_tape_device(&device)?);
}
if let Ok(name) = std::env::var("PROXMOX_TAPE_DRIVE") {
let (config, _digest) = pbs_config::drive::config()?;
let drive: LtoTapeDrive = config.lookup("lto", &name)?;
eprintln!("using device {}", drive.path);
return LtoTapeHandle::new(open_lto_tape_device(&drive.path)?);
return SgTape::new(open_lto_tape_device(&drive.path)?);
}
if let Ok(device) = std::env::var("TAPE") {
eprintln!("using device {}", device);
return LtoTapeHandle::new(open_lto_tape_device(&device)?);
return SgTape::new(open_lto_tape_device(&device)?);
}
let (config, _digest) = pbs_config::drive::config()?;
@ -112,7 +106,7 @@ fn get_tape_handle(param: &Value) -> Result<LtoTapeHandle, Error> {
let name = drive_names[0];
let drive: LtoTapeDrive = config.lookup("lto", &name)?;
eprintln!("using device {}", drive.path);
return LtoTapeHandle::new(open_lto_tape_device(&drive.path)?);
return SgTape::new(open_lto_tape_device(&drive.path)?);
}
bail!("no drive/device specified");
@ -171,7 +165,7 @@ fn bsf(count: usize, param: Value) -> Result<(), Error> {
let mut handle = get_tape_handle(&param)?;
handle.backward_space_count_files(count)?;
handle.space_filemarks(-count.try_into()?)?;
Ok(())
}
@ -202,8 +196,8 @@ fn bsfm(count: usize, param: Value) -> Result<(), Error> {
let mut handle = get_tape_handle(&param)?;
handle.backward_space_count_files(count)?;
handle.forward_space_count_files(1)?;
handle.space_filemarks(-count.try_into()?)?;
handle.space_filemarks(1)?;
Ok(())
}
@ -231,7 +225,7 @@ fn bsr(count: usize, param: Value) -> Result<(), Error> {
let mut handle = get_tape_handle(&param)?;
handle.backward_space_count_records(count)?;
handle.space_blocks(-count.try_into()?)?;
Ok(())
}
@ -355,7 +349,7 @@ fn tape_alert_flags(param: Value) -> Result<(), Error> {
fn eject(param: Value) -> Result<(), Error> {
let mut handle = get_tape_handle(&param)?;
handle.eject_media()?;
handle.eject()?;
Ok(())
}
@ -467,7 +461,7 @@ fn fsf(count: usize, param: Value) -> Result<(), Error> {
let mut handle = get_tape_handle(&param)?;
handle.forward_space_count_files(count)?;
handle.space_filemarks(count.try_into()?)?;
Ok(())
}
@ -497,8 +491,8 @@ fn fsfm(count: usize, param: Value) -> Result<(), Error> {
let mut handle = get_tape_handle(&param)?;
handle.forward_space_count_files(count)?;
handle.backward_space_count_files(1)?;
handle.space_filemarks(count.try_into()?)?;
handle.space_filemarks(-1)?;
Ok(())
}
@ -526,7 +520,7 @@ fn fsr(count: usize, param: Value) -> Result<(), Error> {
let mut handle = get_tape_handle(&param)?;
handle.forward_space_count_records(count)?;
handle.space_blocks(count.try_into()?)?;
Ok(())
}
@ -575,7 +569,7 @@ fn lock(param: Value) -> Result<(), Error> {
let mut handle = get_tape_handle(&param)?;
handle.lock()?;
handle.set_medium_removal(false)?;
Ok(())
}
@ -667,6 +661,7 @@ fn status(param: Value) -> Result<(), Error> {
let output_format = get_output_format(&param);
let mut handle = get_tape_handle(&param)?;
let result = handle.get_drive_and_media_status();
if output_format == "json-pretty" {
@ -712,7 +707,7 @@ fn unlock(param: Value) -> Result<(), Error> {
let mut handle = get_tape_handle(&param)?;
handle.unlock()?;
handle.set_medium_removal(true)?;
Ok(())
}
@ -792,7 +787,7 @@ fn weof(count: Option<usize>, param: Value) -> Result<(), Error> {
let mut handle = get_tape_handle(&param)?;
handle.write_filemarks(count)?;
handle.write_filemarks(count, false)?;
Ok(())
}

16
src/bin/pmtx.rs → pbs-tape/src/bin/pmtx.rs
View File

@ -29,17 +29,11 @@ use pbs_config::drive::complete_changer_name;
use pbs_api_types::{
SCSI_CHANGER_PATH_SCHEMA, CHANGER_NAME_SCHEMA, ScsiTapeChanger, LtoTapeDrive,
};
use proxmox_backup::{
tools::sgutils2::scsi_inquiry,
tape::{
linux_tape_changer_list,
complete_changer_path,
changer::{
ElementStatus,
sg_pt_changer,
},
},
use pbs_tape::{
sgutils2::scsi_inquiry,
ElementStatus,
sg_pt_changer,
linux_list_drives::{complete_changer_path, linux_tape_changer_list},
};
fn get_changer_handle(param: &Value) -> Result<File, Error> {

23
src/tape/file_formats/blocked_reader.rs → pbs-tape/src/blocked_reader.rs
View File

@ -1,14 +1,12 @@
use std::io::Read;
use crate::tape::{
use crate::{
TapeRead,
BlockRead,
BlockReadError,
file_formats::{
PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0,
BlockHeader,
BlockHeaderFlags,
},
PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0,
BlockHeader,
BlockHeaderFlags,
};
/// Read a block stream generated by 'BlockWriter'.
@ -246,15 +244,14 @@ impl <R: BlockRead> Read for BlockedReader<R> {
mod test {
use std::io::Read;
use anyhow::{bail, Error};
use crate::tape::{
use crate::{
TapeWrite,
BlockReadError,
helpers::{EmulateTapeReader, EmulateTapeWriter},
file_formats::{
PROXMOX_TAPE_BLOCK_SIZE,
BlockedReader,
BlockedWriter,
},
EmulateTapeReader,
EmulateTapeWriter,
PROXMOX_TAPE_BLOCK_SIZE,
BlockedReader,
BlockedWriter,
};
fn write_and_verify(data: &[u8]) -> Result<(), Error> {

8
src/tape/file_formats/blocked_writer.rs → pbs-tape/src/blocked_writer.rs
View File

@ -1,12 +1,10 @@
use proxmox::tools::vec;
use crate::tape::{
use crate::{
TapeWrite,
BlockWrite,
file_formats::{
BlockHeader,
BlockHeaderFlags,
},
BlockHeader,
BlockHeaderFlags,
};
/// Assemble and write blocks of data

6
src/tape/helpers/emulate_tape_reader.rs → pbs-tape/src/emulate_tape_reader.rs
View File

@ -2,11 +2,7 @@ use std::io::Read;
use proxmox::tools::io::ReadExt;
use crate::tape::{
BlockRead,
BlockReadError,
file_formats::PROXMOX_TAPE_BLOCK_SIZE,
};
use crate::{BlockRead, BlockReadError, PROXMOX_TAPE_BLOCK_SIZE};
/// Emulate tape read behavior on a normal Reader
///

5
src/tape/helpers/emulate_tape_writer.rs → pbs-tape/src/emulate_tape_writer.rs
View File

@ -1,9 +1,6 @@
use std::io::{self, Write};
use crate::tape::{
BlockWrite,
file_formats::PROXMOX_TAPE_BLOCK_SIZE,
};
use crate::{BlockWrite, PROXMOX_TAPE_BLOCK_SIZE};
/// Emulate tape write behavior on a normal Writer
///

334
pbs-tape/src/lib.rs Normal file
View File

@ -0,0 +1,334 @@
use std::collections::HashSet;
use anyhow::{bail, Error};
use bitflags::bitflags;
use endian_trait::Endian;
use serde::{Serialize, Deserialize};
use serde_json::Value;
use proxmox::tools::Uuid;
use proxmox::api::schema::parse_property_string;
use pbs_api_types::{ScsiTapeChanger, SLOT_ARRAY_SCHEMA};
pub mod linux_list_drives;
pub mod sgutils2;
mod blocked_reader;
pub use blocked_reader::BlockedReader;
mod blocked_writer;
pub use blocked_writer::BlockedWriter;
mod tape_write;
pub use tape_write::*;
mod tape_read;
pub use tape_read::*;
mod emulate_tape_reader;
pub use emulate_tape_reader::EmulateTapeReader;
mod emulate_tape_writer;
pub use emulate_tape_writer::EmulateTapeWriter;
pub mod sg_tape;
pub mod sg_pt_changer;
/// We use 256KB blocksize (always)
pub const PROXMOX_TAPE_BLOCK_SIZE: usize = 256*1024;
// openssl::sha::sha256(b"Proxmox Tape Block Header v1.0")[0..8]
pub const PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0: [u8; 8] = [220, 189, 175, 202, 235, 160, 165, 40];
// openssl::sha::sha256(b"Proxmox Backup Content Header v1.0")[0..8];
pub const PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0: [u8; 8] = [99, 238, 20, 159, 205, 242, 155, 12];
// openssl::sha::sha256(b"Proxmox Backup Tape Label v1.0")[0..8];
pub const PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0: [u8; 8] = [42, 5, 191, 60, 176, 48, 170, 57];
// openssl::sha::sha256(b"Proxmox Backup MediaSet Label v1.0")
pub const PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0: [u8; 8] = [8, 96, 99, 249, 47, 151, 83, 216];
/// Tape Block Header with data payload
///
/// All tape files are written as sequence of blocks.
///
/// Note: this struct is large, never put this on the stack!
/// so we use an unsized type to avoid that.
///
/// Tape data block are always read/written with a fixed size
/// (`PROXMOX_TAPE_BLOCK_SIZE`). But they may contain less data, so the
/// header has an additional size field. For streams of blocks, there
/// is a sequence number (`seq_nr`) which may be use for additional
/// error checking.
#[repr(C,packed)]
pub struct BlockHeader {
/// fixed value `PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0`
pub magic: [u8; 8],
pub flags: BlockHeaderFlags,
/// size as 3 bytes unsigned, little endian
pub size: [u8; 3],
/// block sequence number
pub seq_nr: u32,
pub payload: [u8],
}
bitflags! {
/// Header flags (e.g. `END_OF_STREAM` or `INCOMPLETE`)
pub struct BlockHeaderFlags: u8 {
/// Marks the last block in a stream.
const END_OF_STREAM = 0b00000001;
/// Mark multivolume streams (when set in the last block)
const INCOMPLETE = 0b00000010;
}
}
#[derive(Endian, Copy, Clone, Debug)]
#[repr(C,packed)]
/// Media Content Header
///
/// All tape files start with this header. The header may contain some
/// informational data indicated by `size`.
///
/// `| MediaContentHeader | header data (size) | stream data |`
///
/// Note: The stream data following may be of any size.
pub struct MediaContentHeader {
/// fixed value `PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0`
pub magic: [u8; 8],
/// magic number for the content following
pub content_magic: [u8; 8],
/// unique ID to identify this data stream
pub uuid: [u8; 16],
/// stream creation time
pub ctime: i64,
/// Size of header data
pub size: u32,
/// Part number for multipart archives.
pub part_number: u8,
/// Reserved for future use
pub reserved_0: u8,
/// Reserved for future use
pub reserved_1: u8,
/// Reserved for future use
pub reserved_2: u8,
}
impl MediaContentHeader {
/// Create a new instance with autogenerated Uuid
pub fn new(content_magic: [u8; 8], size: u32) -> Self {
let uuid = *proxmox::tools::uuid::Uuid::generate()
.into_inner();
Self {
magic: PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
content_magic,
uuid,
ctime: proxmox::tools::time::epoch_i64(),
size,
part_number: 0,
reserved_0: 0,
reserved_1: 0,
reserved_2: 0,
}
}
/// Helper to check magic numbers and size constraints
pub fn check(&self, content_magic: [u8; 8], min_size: u32, max_size: u32) -> Result<(), Error> {
if self.magic != PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0 {
bail!("MediaContentHeader: wrong magic");
}
if self.content_magic != content_magic {
bail!("MediaContentHeader: wrong content magic");
}
if self.size < min_size || self.size > max_size {
bail!("MediaContentHeader: got unexpected size");
}
Ok(())
}
/// Returns the content Uuid
pub fn content_uuid(&self) -> Uuid {
Uuid::from(self.uuid)
}
}
impl BlockHeader {
pub const SIZE: usize = PROXMOX_TAPE_BLOCK_SIZE;
/// Allocates a new instance on the heap
pub fn new() -> Box<Self> {
use std::alloc::{alloc_zeroed, Layout};
// align to PAGESIZE, so that we can use it with SG_IO
let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize;
let mut buffer = unsafe {
let ptr = alloc_zeroed(
Layout::from_size_align(Self::SIZE, page_size)
.unwrap(),
);
Box::from_raw(
std::slice::from_raw_parts_mut(ptr, Self::SIZE - 16)
as *mut [u8] as *mut Self
)
};
buffer.magic = PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0;
buffer
}
/// Set the `size` field
pub fn set_size(&mut self, size: usize) {
let size = size.to_le_bytes();
self.size.copy_from_slice(&size[..3]);
}
/// Returns the `size` field
pub fn size(&self) -> usize {
(self.size[0] as usize) + ((self.size[1] as usize)<<8) + ((self.size[2] as usize)<<16)
}
/// Set the `seq_nr` field
pub fn set_seq_nr(&mut self, seq_nr: u32) {
self.seq_nr = seq_nr.to_le();
}
/// Returns the `seq_nr` field
pub fn seq_nr(&self) -> u32 {
u32::from_le(self.seq_nr)
}
}
/// Changer element status.
///
/// Drive and slots may be `Empty`, or contain some media, either
/// with known volume tag `VolumeTag(String)`, or without (`Full`).
#[derive(Serialize, Deserialize, Debug)]
pub enum ElementStatus {
Empty,
Full,
VolumeTag(String),
}
/// Changer drive status.
#[derive(Serialize, Deserialize)]
pub struct DriveStatus {
/// The slot the element was loaded from (if known).
pub loaded_slot: Option<u64>,
/// The status.
pub status: ElementStatus,
/// Drive Identifier (Serial number)
pub drive_serial_number: Option<String>,
/// Drive Vendor
pub vendor: Option<String>,
/// Drive Model
pub model: Option<String>,
/// Element Address
pub element_address: u16,
}
/// Storage element status.
#[derive(Serialize, Deserialize)]
pub struct StorageElementStatus {
/// Flag for Import/Export slots
pub import_export: bool,
/// The status.
pub status: ElementStatus,
/// Element Address
pub element_address: u16,
}
/// Transport element status.
#[derive(Serialize, Deserialize)]
pub struct TransportElementStatus {
/// The status.
pub status: ElementStatus,
/// Element Address
pub element_address: u16,
}
/// Changer status - show drive/slot usage
#[derive(Serialize, Deserialize)]
pub struct MtxStatus {
/// List of known drives
pub drives: Vec<DriveStatus>,
/// List of known storage slots
pub slots: Vec<StorageElementStatus>,
/// Transport elements
///
/// Note: Some libraries do not report transport elements.
pub transports: Vec<TransportElementStatus>,
}
impl MtxStatus {
pub fn slot_address(&self, slot: u64) -> Result<u16, Error> {
if slot == 0 {
bail!("invalid slot number '{}' (slots numbers starts at 1)", slot);
}
if slot > (self.slots.len() as u64) {
bail!("invalid slot number '{}' (max {} slots)", slot, self.slots.len());
}
Ok(self.slots[(slot -1) as usize].element_address)
}
pub fn drive_address(&self, drivenum: u64) -> Result<u16, Error> {
if drivenum >= (self.drives.len() as u64) {
bail!("invalid drive number '{}'", drivenum);
}
Ok(self.drives[drivenum as usize].element_address)
}
pub fn transport_address(&self) -> u16 {
// simply use first transport
// (are there changers exposing more than one?)
// defaults to 0 for changer that do not report transports
self
.transports
.get(0)
.map(|t| t.element_address)
.unwrap_or(0u16)
}
pub fn find_free_slot(&self, import_export: bool) -> Option<u64> {
let mut free_slot = None;
for (i, slot_info) in self.slots.iter().enumerate() {
if slot_info.import_export != import_export {
continue; // skip slots of wrong type
}
if let ElementStatus::Empty = slot_info.status {
free_slot = Some((i+1) as u64);
break;
}
}
free_slot
}
pub fn mark_import_export_slots(&mut self, config: &ScsiTapeChanger) -> Result<(), Error>{
let mut export_slots: HashSet<u64> = HashSet::new();
if let Some(slots) = &config.export_slots {
let slots: Value = parse_property_string(&slots, &SLOT_ARRAY_SCHEMA)?;
export_slots = slots
.as_array()
.unwrap()
.iter()
.filter_map(|v| v.as_u64())
.collect();
}
for (i, entry) in self.slots.iter_mut().enumerate() {
let slot = i as u64 + 1;
if export_slots.contains(&slot) {
entry.import_export = true; // mark as IMPORT/EXPORT
}
}
Ok(())
}
}

63
src/tape/linux_list_drives.rs → pbs-tape/src/linux_list_drives.rs
View File

@ -1,7 +1,13 @@
use std::path::{Path, PathBuf};
use std::collections::HashMap;
use std::fs::{OpenOptions, File};
use std::os::unix::fs::OpenOptionsExt;
use std::os::unix::io::AsRawFd;
use anyhow::{bail, Error};
use anyhow::{bail, format_err, Error};
use nix::fcntl::{fcntl, FcntlArg, OFlag};
use proxmox::sys::error::SysResult;
use pbs_tools::fs::scan_subdir;
use pbs_api_types::{DeviceKind, OptionalDeviceIdentification, TapeDeviceInfo};
@ -248,6 +254,61 @@ pub fn check_drive_path(
Ok(())
}
/// Check for correct Major/Minor numbers
pub fn check_tape_is_lto_tape_device(file: &File) -> Result<(), Error> {
let stat = nix::sys::stat::fstat(file.as_raw_fd())?;
let devnum = stat.st_rdev;
let major = unsafe { libc::major(devnum) };
let _minor = unsafe { libc::minor(devnum) };
if major == 9 {
bail!("not a scsi-generic tape device (cannot use linux tape devices)");
}
if major != 21 {
bail!("not a scsi-generic tape device");
}
Ok(())
}
/// Opens a Lto tape device
///
/// The open call use O_NONBLOCK, but that flag is cleard after open
/// succeeded. This also checks if the device is a non-rewinding tape
/// device.
pub fn open_lto_tape_device(
path: &str,
) -> Result<File, Error> {
let file = OpenOptions::new()
.read(true)
.write(true)
.custom_flags(libc::O_NONBLOCK)
.open(path)?;
// clear O_NONBLOCK from now on.
let flags = fcntl(file.as_raw_fd(), FcntlArg::F_GETFL)
.into_io_result()?;
let mut flags = OFlag::from_bits_truncate(flags);
flags.remove(OFlag::O_NONBLOCK);
fcntl(file.as_raw_fd(), FcntlArg::F_SETFL(flags))
.into_io_result()?;
check_tape_is_lto_tape_device(&file)
.map_err(|err| format_err!("device type check {:?} failed - {}", path, err))?;
Ok(file)
}
// shell completion helper
/// List changer device paths

13
src/tape/changer/sg_pt_changer.rs → pbs-tape/src/sg_pt_changer.rs
View File

@ -1,5 +1,4 @@
//! SCSI changer implementation using libsgutil2
use std::os::unix::prelude::AsRawFd;
use std::io::Read;
use std::collections::HashMap;
@ -14,16 +13,8 @@ use proxmox::tools::io::ReadExt;
use pbs_api_types::ScsiTapeChanger;
use crate::{
tape::{
changer::{
DriveStatus,
ElementStatus,
StorageElementStatus,
TransportElementStatus,
MtxStatus,
},
},
tools::sgutils2::{
ElementStatus,MtxStatus,TransportElementStatus,DriveStatus,StorageElementStatus,
sgutils2::{
SgRaw,
SENSE_KEY_NOT_READY,
ScsiError,

102
src/tape/drive/lto/sg_tape.rs → pbs-tape/src/sg_tape.rs
View File

@ -4,6 +4,7 @@ use std::os::unix::fs::OpenOptionsExt;
use std::os::unix::io::AsRawFd;
use std::path::Path;
use std::convert::TryFrom;
use std::convert::TryInto;
use anyhow::{bail, format_err, Error};
use endian_trait::Endian;
@ -29,19 +30,15 @@ use proxmox::{
tools::io::{ReadExt, WriteExt},
};
use pbs_api_types::{MamAttribute, Lp17VolumeStatistics};
use pbs_api_types::{MamAttribute, Lp17VolumeStatistics, LtoDriveAndMediaStatus};
use crate::{
tape::{
BlockRead,
BlockReadError,
BlockWrite,
file_formats::{
BlockedWriter,
BlockedReader,
},
},
tools::sgutils2::{
BlockRead,
BlockReadError,
BlockWrite,
BlockedWriter,
BlockedReader,
sgutils2::{
SgRaw,
SenseInfo,
ScsiError,
@ -722,6 +719,18 @@ impl SgTape {
BlockedReader::open(reader)
}
/// Set all options we need/want
pub fn set_default_options(&mut self) -> Result<(), Error> {
let compression = Some(true);
let block_length = Some(0); // variable length mode
let buffer_mode = Some(true); // Always use drive buffer
self.set_drive_options(compression, block_length, buffer_mode)?;
Ok(())
}
/// Set important drive options
pub fn set_drive_options(
&mut self,
@ -845,6 +854,77 @@ impl SgTape {
density_code: block_descriptor.density_code,
})
}
/// Get Tape and Media status
pub fn get_drive_and_media_status(&mut self) -> Result<LtoDriveAndMediaStatus, Error> {
let drive_status = self.read_drive_status()?;
let alert_flags = self.tape_alert_flags()
.map(|flags| format!("{:?}", flags))
.ok();
let mut status = LtoDriveAndMediaStatus {
vendor: self.info().vendor.clone(),
product: self.info().product.clone(),
revision: self.info().revision.clone(),
blocksize: drive_status.block_length,
compression: drive_status.compression,
buffer_mode: drive_status.buffer_mode,
density: drive_status.density_code.try_into()?,
alert_flags,
write_protect: None,
file_number: None,
block_number: None,
manufactured: None,
bytes_read: None,
bytes_written: None,
medium_passes: None,
medium_wearout: None,
volume_mounts: None,
};
if self.test_unit_ready().is_ok() {
if drive_status.write_protect {
status.write_protect = Some(drive_status.write_protect);
}
let position = self.position()?;
status.file_number = Some(position.logical_file_id);
status.block_number = Some(position.logical_object_number);
if let Ok(mam) = self.cartridge_memory() {
let usage = mam_extract_media_usage(&mam)?;
status.manufactured = Some(usage.manufactured);
status.bytes_read = Some(usage.bytes_read);
status.bytes_written = Some(usage.bytes_written);
if let Ok(volume_stats) = self.volume_statistics() {
let passes = std::cmp::max(
volume_stats.beginning_of_medium_passes,
volume_stats.middle_of_tape_passes,
);
// assume max. 16000 medium passes
// see: https://en.wikipedia.org/wiki/Linear_Tape-Open
let wearout: f64 = (passes as f64)/(16000.0 as f64);
status.medium_passes = Some(passes);
status.medium_wearout = Some(wearout);
status.volume_mounts = Some(volume_stats.volume_mounts);
}
}
}
Ok(status)
}
}
impl Drop for SgTape {

5
src/tape/drive/lto/sg_tape/encryption.rs → pbs-tape/src/sg_tape/encryption.rs
View File

@ -6,10 +6,7 @@ use endian_trait::Endian;
use proxmox::tools::io::{ReadExt, WriteExt};
use crate::tools::sgutils2::{
SgRaw,
alloc_page_aligned_buffer,
};
use crate::sgutils2::{SgRaw, alloc_page_aligned_buffer};
/// Test if drive supports hardware encryption
///

7
src/tape/drive/lto/sg_tape/mam.rs → pbs-tape/src/sg_tape/mam.rs
View File

@ -9,10 +9,9 @@ use proxmox::tools::io::ReadExt;
use pbs_api_types::MamAttribute;
use crate::{
tools::sgutils2::SgRaw,
tape::drive::lto::TapeAlertFlags,
};
use crate::sgutils2::SgRaw;
use super::TapeAlertFlags;
// Read Medium auxiliary memory attributes (MAM)
// see IBM SCSI reference: https://www-01.ibm.com/support/docview.wss?uid=ssg1S7003556&aid=1

2
src/tape/drive/lto/sg_tape/report_density.rs → pbs-tape/src/sg_tape/report_density.rs
View File

@ -5,7 +5,7 @@ use std::os::unix::io::AsRawFd;
use proxmox::tools::io::ReadExt;
use crate::tools::sgutils2::SgRaw;
use crate::sgutils2::SgRaw;
#[repr(C, packed)]
#[derive(Endian)]

2
src/tape/drive/lto/sg_tape/tape_alert_flags.rs → pbs-tape/src/sg_tape/tape_alert_flags.rs
View File

@ -5,7 +5,7 @@ use anyhow::{bail, format_err, Error};
use proxmox::tools::io::ReadExt;
use crate::tools::sgutils2::SgRaw;
use crate::sgutils2::SgRaw;
bitflags::bitflags!{

2
src/tape/drive/lto/sg_tape/volume_statistics.rs → pbs-tape/src/sg_tape/volume_statistics.rs
View File

@ -8,7 +8,7 @@ use proxmox::tools::io::ReadExt;
use pbs_api_types::Lp17VolumeStatistics;
use crate::tools::sgutils2::SgRaw;
use crate::sgutils2::SgRaw;
/// SCSI command to query volume statistics
///

0
src/tools/sgutils2.rs → pbs-tape/src/sgutils2.rs
View File

0
src/tape/tape_read.rs → pbs-tape/src/tape_read.rs
View File

2
src/tape/tape_write.rs → pbs-tape/src/tape_write.rs
View File

@ -1,6 +1,6 @@
use endian_trait::Endian;
use crate::tape::file_formats::MediaContentHeader;
use crate::MediaContentHeader;
/// Write trait for tape devices
///

8
src/api2/config/changer.rs
View File

@ -16,13 +16,7 @@ use pbs_api_types::{
PRIV_TAPE_AUDIT, PRIV_TAPE_MODIFY,
};
use pbs_config::CachedUserInfo;
use crate::{
tape::{
linux_tape_changer_list,
check_drive_path,
},
};
use pbs_tape::linux_list_drives::{linux_tape_changer_list, check_drive_path};
#[api(
protected: true,

7
src/api2/config/drive.rs
View File

@ -10,12 +10,7 @@ use pbs_api_types::{
};
use pbs_config::CachedUserInfo;
use crate::{
tape::{
lto_tape_device_list,
check_drive_path,
},
};
use pbs_tape::linux_list_drives::{lto_tape_device_list, check_drive_path};
#[api(
protected: true,

7
src/api2/tape/changer.rs
View File

@ -12,20 +12,21 @@ use pbs_api_types::{
CHANGER_NAME_SCHEMA, PRIV_TAPE_AUDIT, PRIV_TAPE_READ,
};
use pbs_config::CachedUserInfo;
use pbs_tape::{
ElementStatus,
linux_list_drives::{lookup_device_identification, linux_tape_changer_list},
};
use crate::{
tape::{
TAPE_STATUS_DIR,
Inventory,
linux_tape_changer_list,
changer::{
OnlineStatusMap,
ElementStatus,
ScsiMediaChange,
mtx_status_to_online_set,
},
drive::get_tape_device_state,
lookup_device_identification,
},
};

10
src/api2/tape/drive.rs
View File

@ -31,6 +31,11 @@ use pbs_api_types::{
use pbs_datastore::task_log;
use pbs_api_types::{PRIV_TAPE_AUDIT, PRIV_TAPE_READ, PRIV_TAPE_WRITE};
use pbs_config::CachedUserInfo;
use pbs_tape::{
BlockReadError,
sg_tape::tape_alert_flags_critical,
linux_list_drives::{lto_tape_device_list, lookup_device_identification, open_lto_tape_device},
};
use crate::{
api2::tape::restore::{
@ -43,12 +48,9 @@ use crate::{
Inventory,
MediaCatalog,
MediaId,
BlockReadError,
lock_media_set,
lock_media_pool,
lock_unassigned_media_pool,
lto_tape_device_list,
lookup_device_identification,
file_formats::{
MediaLabel,
MediaSetLabel,
@ -56,7 +58,6 @@ use crate::{
drive::{
TapeDriver,
LtoTapeHandle,
open_lto_tape_device,
open_lto_tape_drive,
media_changer,
required_media_changer,
@ -64,7 +65,6 @@ use crate::{
lock_tape_device,
set_tape_device_state,
get_tape_device_state,
tape_alert_flags_critical,
},
changer::update_changer_online_status,
},

8
src/api2/tape/mod.rs
View File

@ -13,13 +13,7 @@ use proxmox::{
};
use pbs_api_types::TapeDeviceInfo;
use crate::{
tape::{
lto_tape_device_list,
linux_tape_changer_list,
},
};
use pbs_tape::linux_list_drives::{lto_tape_device_list, linux_tape_changer_list};
pub mod drive;
pub mod changer;

8
src/api2/tape/restore.rs
View File

@ -42,6 +42,10 @@ use pbs_datastore::index::IndexFile;
use pbs_datastore::manifest::{archive_type, ArchiveType, BackupManifest, MANIFEST_BLOB_NAME};
use pbs_datastore::task::TaskState;
use pbs_config::CachedUserInfo;
use pbs_tape::{
TapeRead, BlockReadError, MediaContentHeader,
PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
};
use crate::{
tools::ParallelHandler,
@ -52,8 +56,6 @@ use crate::{
},
tape::{
TAPE_STATUS_DIR,
TapeRead,
BlockReadError,
MediaId,
MediaSet,
MediaCatalog,
@ -65,11 +67,9 @@ use crate::{
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1,
PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1,
PROXMOX_BACKUP_CATALOG_ARCHIVE_MAGIC_1_0,
MediaContentHeader,
ChunkArchiveHeader,
ChunkArchiveDecoder,
SnapshotArchiveHeader,

6
src/bin/proxmox-tape.rs
View File

@ -30,11 +30,13 @@ use pbs_api_types::{
DRIVE_NAME_SCHEMA, MEDIA_LABEL_SCHEMA, MEDIA_POOL_NAME_SCHEMA,
TAPE_RESTORE_SNAPSHOT_SCHEMA,
};
use pbs_tape::{
PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0, BlockReadError, MediaContentHeader,
};
use proxmox_backup::{
api2,
tape::{
BlockReadError,
drive::{
open_drive,
lock_tape_device,
@ -44,8 +46,6 @@ use proxmox_backup::{
complete_media_set_uuid,
complete_media_set_snapshots,
file_formats::{
PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
MediaContentHeader,
proxmox_tape_magic_to_text,
},
},

10
src/bin/proxmox_tape/changer.rs
View File

@ -18,13 +18,9 @@ use pbs_config::drive::{
use pbs_api_types::CHANGER_NAME_SCHEMA;
use proxmox_backup::{
api2,
tape::{
complete_changer_path,
drive::media_changer,
},
};
use pbs_tape::linux_list_drives::{complete_changer_path};
use proxmox_backup::{api2, tape::drive::media_changer};
pub fn lookup_changer_name(
param: &Value,

4
src/bin/proxmox_tape/drive.rs
View File

@ -18,7 +18,9 @@ use pbs_config::drive::{
complete_lto_drive_name,
};
use proxmox_backup::{api2, tape::complete_drive_path};
use pbs_tape::linux_list_drives::{complete_drive_path};
use proxmox_backup::api2;
pub fn drive_commands() -> CommandLineInterface {

4
src/bin/sg-tape-cmd.rs
View File

@ -23,14 +23,14 @@ use pbs_api_types::{
MEDIA_SET_UUID_SCHEMA, LtoTapeDrive,
};
use pbs_tape::linux_list_drives::{open_lto_tape_device, check_tape_is_lto_tape_device};
use proxmox_backup::{
tape::{
drive::{
TapeDriver,
LtoTapeHandle,
open_lto_tape_device,
open_lto_tape_drive,
check_tape_is_lto_tape_device,
},
},
};

147
src/tape/changer/mod.rs
View File

@ -1,160 +1,19 @@
//! Media changer implementation (SCSI media changer)
pub mod sg_pt_changer;
pub mod mtx;
mod online_status_map;
pub use online_status_map::*;
use std::collections::HashSet;
use std::path::PathBuf;
use anyhow::{bail, Error};
use serde::{Serialize, Deserialize};
use serde_json::Value;
use proxmox::{
api::schema::parse_property_string,
tools::fs::{
CreateOptions,
replace_file,
file_read_optional_string,
},
};
use proxmox::tools::fs::{CreateOptions, replace_file, file_read_optional_string};
use pbs_api_types::{SLOT_ARRAY_SCHEMA, ScsiTapeChanger, LtoTapeDrive};
use pbs_api_types::{ScsiTapeChanger, LtoTapeDrive};
/// Changer element status.
///
/// Drive and slots may be `Empty`, or contain some media, either
/// with known volume tag `VolumeTag(String)`, or without (`Full`).
#[derive(Serialize, Deserialize, Debug)]
pub enum ElementStatus {
Empty,
Full,
VolumeTag(String),
}
/// Changer drive status.
#[derive(Serialize, Deserialize)]
pub struct DriveStatus {
/// The slot the element was loaded from (if known).
pub loaded_slot: Option<u64>,
/// The status.
pub status: ElementStatus,
/// Drive Identifier (Serial number)
pub drive_serial_number: Option<String>,
/// Drive Vendor
pub vendor: Option<String>,
/// Drive Model
pub model: Option<String>,
/// Element Address
pub element_address: u16,
}
/// Storage element status.
#[derive(Serialize, Deserialize)]
pub struct StorageElementStatus {
/// Flag for Import/Export slots
pub import_export: bool,
/// The status.
pub status: ElementStatus,
/// Element Address
pub element_address: u16,
}
/// Transport element status.
#[derive(Serialize, Deserialize)]
pub struct TransportElementStatus {
/// The status.
pub status: ElementStatus,
/// Element Address
pub element_address: u16,
}
/// Changer status - show drive/slot usage
#[derive(Serialize, Deserialize)]
pub struct MtxStatus {
/// List of known drives
pub drives: Vec<DriveStatus>,
/// List of known storage slots
pub slots: Vec<StorageElementStatus>,
/// Transport elements
///
/// Note: Some libraries do not report transport elements.
pub transports: Vec<TransportElementStatus>,
}
impl MtxStatus {
pub fn slot_address(&self, slot: u64) -> Result<u16, Error> {
if slot == 0 {
bail!("invalid slot number '{}' (slots numbers starts at 1)", slot);
}
if slot > (self.slots.len() as u64) {
bail!("invalid slot number '{}' (max {} slots)", slot, self.slots.len());
}
Ok(self.slots[(slot -1) as usize].element_address)
}
pub fn drive_address(&self, drivenum: u64) -> Result<u16, Error> {
if drivenum >= (self.drives.len() as u64) {
bail!("invalid drive number '{}'", drivenum);
}
Ok(self.drives[drivenum as usize].element_address)
}
pub fn transport_address(&self) -> u16 {
// simply use first transport
// (are there changers exposing more than one?)
// defaults to 0 for changer that do not report transports
self
.transports
.get(0)
.map(|t| t.element_address)
.unwrap_or(0u16)
}
pub fn find_free_slot(&self, import_export: bool) -> Option<u64> {
let mut free_slot = None;
for (i, slot_info) in self.slots.iter().enumerate() {
if slot_info.import_export != import_export {
continue; // skip slots of wrong type
}
if let ElementStatus::Empty = slot_info.status {
free_slot = Some((i+1) as u64);
break;
}
}
free_slot
}
pub fn mark_import_export_slots(&mut self, config: &ScsiTapeChanger) -> Result<(), Error>{
let mut export_slots: HashSet<u64> = HashSet::new();
if let Some(slots) = &config.export_slots {
let slots: Value = parse_property_string(&slots, &SLOT_ARRAY_SCHEMA)?;
export_slots = slots
.as_array()
.unwrap()
.iter()
.filter_map(|v| v.as_u64())
.collect();
}
for (i, entry) in self.slots.iter_mut().enumerate() {
let slot = i as u64 + 1;
if export_slots.contains(&slot) {
entry.import_export = true; // mark as IMPORT/EXPORT
}
}
Ok(())
}
}
use pbs_tape::{sg_pt_changer, MtxStatus, ElementStatus};
/// Interface to SCSI changer devices
pub trait ScsiMediaChange {

4
src/tape/changer/mtx/mtx_wrapper.rs
View File

@ -2,11 +2,11 @@ use anyhow::Error;
use pbs_tools::run_command;
use pbs_api_types::ScsiTapeChanger;
use pbs_tape::MtxStatus;
use crate::{
tape::changer::{
MtxStatus,
mtx::parse_mtx_status,
mtx::parse_mtx_status,
},
};

20
src/tape/changer/mtx/parse_mtx_status.rs
View File

@ -1,20 +1,12 @@
use anyhow::Error;
use nom::{
bytes::complete::{take_while, tag},
};
use nom::bytes::complete::{take_while, tag};
use crate::{
tools::nom::{
parse_complete, multispace0, multispace1, parse_u64,
parse_failure, parse_error, IResult,
},
tape::changer::{
ElementStatus,
MtxStatus,
DriveStatus,
StorageElementStatus,
},
use pbs_tape::{ElementStatus, MtxStatus, DriveStatus, StorageElementStatus};
use crate::tools::nom::{
parse_complete, multispace0, multispace1, parse_u64,
parse_failure, parse_error, IResult,
};

14
src/tape/changer/online_status_map.rs
View File

@ -7,18 +7,10 @@ use proxmox::tools::Uuid;
use proxmox::api::section_config::SectionConfigData;
use pbs_api_types::{VirtualTapeDrive, ScsiTapeChanger};
use pbs_tape::{ElementStatus, MtxStatus};
use crate::{
tape::{
Inventory,
changer::{
MediaChange,
MtxStatus,
ElementStatus,
ScsiMediaChange,
},
},
};
use crate::tape::Inventory;
use crate::tape::changer::{MediaChange, ScsiMediaChange};
/// Helper to update media online status
///

158
src/tape/drive/lto/mod.rs
View File

@ -11,41 +11,29 @@
//!
//! - unability to detect EOT (you just get EIO)
mod sg_tape;
pub use sg_tape::*;
use std::fs::{OpenOptions, File};
use std::os::unix::fs::OpenOptionsExt;
use std::fs::File;
use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
use std::convert::TryInto;
use anyhow::{bail, format_err, Error};
use nix::fcntl::{fcntl, FcntlArg, OFlag};
use proxmox::{
tools::Uuid,
sys::error::SysResult,
};
use proxmox::tools::Uuid;
use pbs_api_types::{
Fingerprint, MamAttribute, LtoDriveAndMediaStatus, LtoTapeDrive, Lp17VolumeStatistics,
};
use pbs_config::key_config::KeyConfig;
use pbs_tools::run_command;
use pbs_tape::{
TapeWrite, TapeRead, BlockReadError, MediaContentHeader,
sg_tape::{SgTape, TapeAlertFlags},
linux_list_drives::open_lto_tape_device,
};
use crate::{
tape::{
TapeRead,
TapeWrite,
BlockReadError,
drive::{
TapeDriver,
},
file_formats::{
PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
MediaSetLabel,
MediaContentHeader,
},
drive::TapeDriver,
file_formats::{PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, MediaSetLabel},
},
};
@ -94,13 +82,7 @@ impl LtoTapeHandle {
/// Set all options we need/want
pub fn set_default_options(&mut self) -> Result<(), Error> {
let compression = Some(true);
let block_length = Some(0); // variable length mode
let buffer_mode = Some(true); // Always use drive buffer
self.set_drive_options(compression, block_length, buffer_mode)?;
self.sg_tape.set_default_options()?;
Ok(())
}
@ -121,72 +103,7 @@ impl LtoTapeHandle {
/// Get Tape and Media status
pub fn get_drive_and_media_status(&mut self) -> Result<LtoDriveAndMediaStatus, Error> {
let drive_status = self.sg_tape.read_drive_status()?;
let alert_flags = self.tape_alert_flags()
.map(|flags| format!("{:?}", flags))
.ok();
let mut status = LtoDriveAndMediaStatus {
vendor: self.sg_tape.info().vendor.clone(),
product: self.sg_tape.info().product.clone(),
revision: self.sg_tape.info().revision.clone(),
blocksize: drive_status.block_length,
compression: drive_status.compression,
buffer_mode: drive_status.buffer_mode,
density: drive_status.density_code.try_into()?,
alert_flags,
write_protect: None,
file_number: None,
block_number: None,
manufactured: None,
bytes_read: None,
bytes_written: None,
medium_passes: None,
medium_wearout: None,
volume_mounts: None,
};
if self.sg_tape.test_unit_ready().is_ok() {
if drive_status.write_protect {
status.write_protect = Some(drive_status.write_protect);
}
let position = self.sg_tape.position()?;
status.file_number = Some(position.logical_file_id);
status.block_number = Some(position.logical_object_number);
if let Ok(mam) = self.cartridge_memory() {
let usage = mam_extract_media_usage(&mam)?;
status.manufactured = Some(usage.manufactured);
status.bytes_read = Some(usage.bytes_read);
status.bytes_written = Some(usage.bytes_written);
if let Ok(volume_stats) = self.volume_statistics() {
let passes = std::cmp::max(
volume_stats.beginning_of_medium_passes,
volume_stats.middle_of_tape_passes,
);
// assume max. 16000 medium passes
// see: https://en.wikipedia.org/wiki/Linear_Tape-Open
let wearout: f64 = (passes as f64)/(16000.0 as f64);
status.medium_passes = Some(passes);
status.medium_wearout = Some(wearout);
status.volume_mounts = Some(volume_stats.volume_mounts);
}
}
}
Ok(status)
self.sg_tape.get_drive_and_media_status()
}
pub fn forward_space_count_files(&mut self, count: usize) -> Result<(), Error> {
@ -413,59 +330,6 @@ impl TapeDriver for LtoTapeHandle {
}
}
/// Check for correct Major/Minor numbers
pub fn check_tape_is_lto_tape_device(file: &File) -> Result<(), Error> {
let stat = nix::sys::stat::fstat(file.as_raw_fd())?;
let devnum = stat.st_rdev;
let major = unsafe { libc::major(devnum) };
let _minor = unsafe { libc::minor(devnum) };
if major == 9 {
bail!("not a scsi-generic tape device (cannot use linux tape devices)");
}
if major != 21 {
bail!("not a scsi-generic tape device");
}
Ok(())
}
/// Opens a Lto tape device
///
/// The open call use O_NONBLOCK, but that flag is cleard after open
/// succeeded. This also checks if the device is a non-rewinding tape
/// device.
pub fn open_lto_tape_device(
path: &str,
) -> Result<File, Error> {
let file = OpenOptions::new()
.read(true)
.write(true)
.custom_flags(libc::O_NONBLOCK)
.open(path)?;
// clear O_NONBLOCK from now on.
let flags = fcntl(file.as_raw_fd(), FcntlArg::F_GETFL)
.into_io_result()?;
let mut flags = OFlag::from_bits_truncate(flags);
flags.remove(OFlag::O_NONBLOCK);
fcntl(file.as_raw_fd(), FcntlArg::F_SETFL(flags))
.into_io_result()?;
check_tape_is_lto_tape_device(&file)
.map_err(|err| format_err!("device type check {:?} failed - {}", path, err))?;
Ok(file)
}
fn run_sg_tape_cmd(subcmd: &str, args: &[&str], fd: RawFd) -> Result<String, Error> {
let mut command = std::process::Command::new(
"/usr/lib/x86_64-linux-gnu/proxmox-backup/sg-tape-cmd");

10
src/tape/drive/mod.rs
View File

@ -33,26 +33,26 @@ use pbs_config::key_config::KeyConfig;
use pbs_datastore::task::TaskState;
use pbs_datastore::task_log;
use pbs_tape::{
TapeWrite, TapeRead, BlockReadError, MediaContentHeader,
sg_tape::TapeAlertFlags,
};
use crate::{
server::{
send_load_media_email,
WorkerTask,
},
tape::{
TapeWrite,
TapeRead,
BlockReadError,
MediaId,
drive::{
virtual_tape::open_virtual_tape_drive,
lto::TapeAlertFlags,
},
file_formats::{
PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0,
PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
MediaLabel,
MediaSetLabel,
MediaContentHeader,
},
changer::{
MediaChange,

38
src/tape/drive/virtual_tape.rs
View File

@ -11,33 +11,31 @@ use proxmox::tools::{
};
use pbs_config::key_config::KeyConfig;
use pbs_tape::{
TapeWrite,
TapeRead,
BlockedReader,
BlockedWriter,
BlockReadError,
MtxStatus,
DriveStatus,
ElementStatus,
StorageElementStatus,
MediaContentHeader,
EmulateTapeReader,
EmulateTapeWriter,
};
use crate::{
tape::{
TapeWrite,
TapeRead,
BlockReadError,
changer::{
MediaChange,
MtxStatus,
DriveStatus,
ElementStatus,
StorageElementStatus,
},
drive::{
VirtualTapeDrive,
TapeDriver,
drive::{
VirtualTapeDrive,
TapeDriver,
MediaChange,
},
file_formats::{
MediaSetLabel,
MediaContentHeader,
PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
BlockedReader,
BlockedWriter,
},
helpers::{
EmulateTapeReader,
EmulateTapeWriter,
},
},
};

8
src/tape/file_formats/catalog_archive.rs
View File

@ -6,13 +6,15 @@ use proxmox::{
tools::Uuid,
};
use pbs_tape::{
PROXMOX_TAPE_BLOCK_SIZE,
TapeWrite, MediaContentHeader,
};
use crate::{
tape::{
TapeWrite,
file_formats::{
PROXMOX_TAPE_BLOCK_SIZE,
PROXMOX_BACKUP_CATALOG_ARCHIVE_MAGIC_1_0,
MediaContentHeader,
CatalogArchiveHeader,
},
},

19
src/tape/file_formats/chunk_archive.rs
View File

@ -9,17 +9,16 @@ use proxmox::tools::{
};
use pbs_datastore::DataBlob;
use pbs_tape::{
PROXMOX_TAPE_BLOCK_SIZE,
TapeWrite, MediaContentHeader,
};
use crate::tape::{
TapeWrite,
file_formats::{
PROXMOX_TAPE_BLOCK_SIZE,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1,
PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
MediaContentHeader,
ChunkArchiveHeader,
ChunkArchiveEntryHeader,
},
use crate::tape::file_formats::{
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1,
PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
ChunkArchiveHeader,
ChunkArchiveEntryHeader,
};
/// Writes chunk archives to tape.

165
src/tape/file_formats/mod.rs
View File

@ -3,8 +3,6 @@
use std::collections::HashMap;
use anyhow::{bail, Error};
use bitflags::bitflags;
use endian_trait::Endian;
use serde::{Deserialize, Serialize};
@ -12,12 +10,6 @@ use proxmox::tools::Uuid;
use pbs_api_types::Fingerprint;
mod blocked_reader;
pub use blocked_reader::*;
mod blocked_writer;
pub use blocked_writer::*;
mod chunk_archive;
pub use chunk_archive::*;
@ -33,14 +25,6 @@ pub use multi_volume_writer::*;
mod multi_volume_reader;
pub use multi_volume_reader::*;
/// We use 256KB blocksize (always)
pub const PROXMOX_TAPE_BLOCK_SIZE: usize = 256*1024;
// openssl::sha::sha256(b"Proxmox Tape Block Header v1.0")[0..8]
pub const PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0: [u8; 8] = [220, 189, 175, 202, 235, 160, 165, 40];
// openssl::sha::sha256(b"Proxmox Backup Content Header v1.0")[0..8];
pub const PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0: [u8; 8] = [99, 238, 20, 159, 205, 242, 155, 12];
// openssl::sha::sha256(b"Proxmox Backup Tape Label v1.0")[0..8];
pub const PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0: [u8; 8] = [42, 5, 191, 60, 176, 48, 170, 57];
// openssl::sha::sha256(b"Proxmox Backup MediaSet Label v1.0")
@ -84,109 +68,6 @@ pub fn proxmox_tape_magic_to_text(magic: &[u8; 8]) -> Option<String> {
PROXMOX_TAPE_CONTENT_NAME.get(magic).map(|s| String::from(*s))
}
/// Tape Block Header with data payload
///
/// All tape files are written as sequence of blocks.
///
/// Note: this struct is large, never put this on the stack!
/// so we use an unsized type to avoid that.
///
/// Tape data block are always read/written with a fixed size
/// (`PROXMOX_TAPE_BLOCK_SIZE`). But they may contain less data, so the
/// header has an additional size field. For streams of blocks, there
/// is a sequence number (`seq_nr`) which may be use for additional
/// error checking.
#[repr(C,packed)]
pub struct BlockHeader {
/// fixed value `PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0`
pub magic: [u8; 8],
pub flags: BlockHeaderFlags,
/// size as 3 bytes unsigned, little endian
pub size: [u8; 3],
/// block sequence number
pub seq_nr: u32,
pub payload: [u8],
}
bitflags! {
/// Header flags (e.g. `END_OF_STREAM` or `INCOMPLETE`)
pub struct BlockHeaderFlags: u8 {
/// Marks the last block in a stream.
const END_OF_STREAM = 0b00000001;
/// Mark multivolume streams (when set in the last block)
const INCOMPLETE = 0b00000010;
}
}
#[derive(Endian, Copy, Clone, Debug)]
#[repr(C,packed)]
/// Media Content Header
///
/// All tape files start with this header. The header may contain some
/// informational data indicated by `size`.
///
/// `| MediaContentHeader | header data (size) | stream data |`
///
/// Note: The stream data following may be of any size.
pub struct MediaContentHeader {
/// fixed value `PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0`
pub magic: [u8; 8],
/// magic number for the content following
pub content_magic: [u8; 8],
/// unique ID to identify this data stream
pub uuid: [u8; 16],
/// stream creation time
pub ctime: i64,
/// Size of header data
pub size: u32,
/// Part number for multipart archives.
pub part_number: u8,
/// Reserved for future use
pub reserved_0: u8,
/// Reserved for future use
pub reserved_1: u8,
/// Reserved for future use
pub reserved_2: u8,
}
impl MediaContentHeader {
/// Create a new instance with autogenerated Uuid
pub fn new(content_magic: [u8; 8], size: u32) -> Self {
let uuid = *proxmox::tools::uuid::Uuid::generate()
.into_inner();
Self {
magic: PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
content_magic,
uuid,
ctime: proxmox::tools::time::epoch_i64(),
size,
part_number: 0,
reserved_0: 0,
reserved_1: 0,
reserved_2: 0,
}
}
/// Helper to check magic numbers and size constraints
pub fn check(&self, content_magic: [u8; 8], min_size: u32, max_size: u32) -> Result<(), Error> {
if self.magic != PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0 {
bail!("MediaContentHeader: wrong magic");
}
if self.content_magic != content_magic {
bail!("MediaContentHeader: wrong content magic");
}
if self.size < min_size || self.size > max_size {
bail!("MediaContentHeader: got unexpected size");
}
Ok(())
}
/// Returns the content Uuid
pub fn content_uuid(&self) -> Uuid {
Uuid::from(self.uuid)
}
}
#[derive(Deserialize, Serialize)]
/// Header for chunk archives
@ -280,49 +161,3 @@ impl MediaSetLabel {
}
}
impl BlockHeader {
pub const SIZE: usize = PROXMOX_TAPE_BLOCK_SIZE;
/// Allocates a new instance on the heap
pub fn new() -> Box<Self> {
use std::alloc::{alloc_zeroed, Layout};
// align to PAGESIZE, so that we can use it with SG_IO
let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize;
let mut buffer = unsafe {
let ptr = alloc_zeroed(
Layout::from_size_align(Self::SIZE, page_size)
.unwrap(),
);
Box::from_raw(
std::slice::from_raw_parts_mut(ptr, Self::SIZE - 16)
as *mut [u8] as *mut Self
)
};
buffer.magic = PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0;
buffer
}
/// Set the `size` field
pub fn set_size(&mut self, size: usize) {
let size = size.to_le_bytes();
self.size.copy_from_slice(&size[..3]);
}
/// Returns the `size` field
pub fn size(&self) -> usize {
(self.size[0] as usize) + ((self.size[1] as usize)<<8) + ((self.size[2] as usize)<<16)
}
/// Set the `seq_nr` field
pub fn set_seq_nr(&mut self, seq_nr: u32) {
self.seq_nr = seq_nr.to_le();
}
/// Returns the `seq_nr` field
pub fn seq_nr(&self) -> u32 {
u32::from_le(self.seq_nr)
}
}

5
src/tape/file_formats/multi_volume_reader.rs
View File

@ -4,10 +4,7 @@ use anyhow::{bail, Error};
use proxmox::tools::io::ReadExt;
use crate::tape::{
TapeRead,
file_formats::MediaContentHeader,
};
use pbs_tape::{TapeRead, MediaContentHeader};
/// Read multi volume data streams written by `MultiVolumeWriter`
///

5
src/tape/file_formats/multi_volume_writer.rs
View File

@ -2,10 +2,7 @@ use anyhow::Error;
use proxmox::tools::Uuid;
use crate::tape::{
TapeWrite,
file_formats::MediaContentHeader,
};
use pbs_tape::{TapeWrite, MediaContentHeader};
/// Writes data streams using multiple volumes
///

8
src/tape/file_formats/snapshot_archive.rs
View File

@ -7,13 +7,15 @@ use proxmox::{
tools::Uuid,
};
use pbs_tape::{
PROXMOX_TAPE_BLOCK_SIZE,
TapeWrite, MediaContentHeader,
};
use crate::tape::{
TapeWrite,
SnapshotReader,
file_formats::{
PROXMOX_TAPE_BLOCK_SIZE,
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1,
MediaContentHeader,
SnapshotArchiveHeader,
},
};

6
src/tape/helpers/mod.rs
View File

@ -1,8 +1,2 @@
mod emulate_tape_writer;
pub use emulate_tape_writer::*;
mod emulate_tape_reader;
pub use emulate_tape_reader::*;
mod snapshot_reader;
pub use snapshot_reader::*;

9
src/tape/mod.rs
View File

@ -14,12 +14,6 @@ mod test;
pub mod file_formats;
mod tape_write;
pub use tape_write::*;
mod tape_read;
pub use tape_read::*;
mod helpers;
pub use helpers::*;
@ -29,9 +23,6 @@ pub use media_set::*;
mod inventory;
pub use inventory::*;
mod linux_list_drives;
pub use linux_list_drives::*;
pub mod changer;
pub mod drive;

6
src/tape/pool_writer/mod.rs
View File

@ -15,6 +15,10 @@ use proxmox::tools::Uuid;
use pbs_datastore::task_log;
use pbs_config::tape_encryption_keys::load_key_configs;
use pbs_tape::{
TapeWrite,
sg_tape::tape_alert_flags_critical,
};
use crate::{
backup::{
@ -25,7 +29,6 @@ use crate::{
TAPE_STATUS_DIR,
MAX_CHUNK_ARCHIVE_SIZE,
COMMIT_BLOCK_SIZE,
TapeWrite,
SnapshotReader,
MediaPool,
MediaId,
@ -39,7 +42,6 @@ use crate::{
drive::{
TapeDriver,
request_and_load_media,
tape_alert_flags_critical,
media_changer,
},
},

1
src/tools/mod.rs
View File

@ -33,7 +33,6 @@ pub mod statistics;
pub mod subscription;
pub mod systemd;
pub mod ticket;
pub mod sgutils2;
pub mod parallel_handler;
pub use parallel_handler::ParallelHandler;