use UPID and systemd helpers from proxmox 0.13.4

This commit is contained in:
Dietmar Maurer 2021-09-22 12:46:44 +02:00
parent 0a33fba49c
commit 81867f0539
23 changed files with 86 additions and 318 deletions

View File

@ -94,7 +94,7 @@ zstd = { version = "0.6", features = [ "bindgen" ] }
pathpatterns = "0.1.2"
pxar = { version = "0.10.1", features = [ "tokio-io" ] }
proxmox = { version = "0.13.3", features = [ "sortable-macro", "api-macro", "cli", "router", "tfa" ] }
proxmox = { version = "0.13.4", features = [ "sortable-macro", "api-macro", "cli", "router", "tfa" ] }
proxmox-acme-rs = "0.2.1"
proxmox-apt = "0.7.0"
proxmox-http = { version = "0.4.0", features = [ "client", "http-helpers", "websocket" ] }

View File

@ -14,7 +14,7 @@ openssl = "0.10"
regex = "1.2"
serde = { version = "1.0", features = ["derive"] }
proxmox = { version = "0.13.3", default-features = false, features = [ "api-macro" ] }
proxmox = { version = "0.13.4", default-features = false, features = [ "api-macro" ] }
proxmox-systemd = { path = "../proxmox-systemd" }
pbs-tools = { path = "../pbs-tools" }

View File

@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
use anyhow::bail;
use proxmox::api::api;
use proxmox::api::schema::{ApiStringFormat, ArraySchema, Schema, StringSchema};
use proxmox::api::schema::{ApiStringFormat, ApiType, ArraySchema, Schema, StringSchema, ReturnType};
use proxmox::const_regex;
use proxmox::{IPRE, IPRE_BRACKET, IPV4OCTET, IPV4RE, IPV6H16, IPV6LS32, IPV6RE};
@ -60,8 +60,7 @@ pub use userid::{PROXMOX_GROUP_ID_SCHEMA, PROXMOX_TOKEN_ID_SCHEMA, PROXMOX_TOKEN
mod user;
pub use user::*;
pub mod upid;
pub use upid::*;
pub use proxmox::api::upid::*;
mod crypto;
pub use crypto::{CryptMode, Fingerprint};
@ -397,3 +396,57 @@ pub enum NodePowerCommand {
/// Shutdown the server
Shutdown,
}
#[api()]
#[derive(Eq, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum TaskStateType {
/// Ok
OK,
/// Warning
Warning,
/// Error
Error,
/// Unknown
Unknown,
}
#[api(
properties: {
upid: { schema: UPID::API_SCHEMA },
},
)]
#[derive(Serialize, Deserialize)]
/// Task properties.
pub struct TaskListItem {
pub upid: String,
/// The node name where the task is running on.
pub node: String,
/// The Unix PID
pub pid: i64,
/// The task start time (Epoch)
pub pstart: u64,
/// The task start time (Epoch)
pub starttime: i64,
/// Worker type (arbitrary ASCII string)
pub worker_type: String,
/// Worker ID (arbitrary ASCII string)
pub worker_id: Option<String>,
/// The authenticated entity who started the task
pub user: String,
/// The task end time (Epoch)
#[serde(skip_serializing_if="Option::is_none")]
pub endtime: Option<i64>,
/// Task end status
#[serde(skip_serializing_if="Option::is_none")]
pub status: Option<String>,
}
pub const NODE_TASKS_LIST_TASKS_RETURN_TYPE: ReturnType = ReturnType {
optional: false,
schema: &ArraySchema::new(
"A list of tasks.",
&TaskListItem::API_SCHEMA,
).schema(),
};

View File

@ -1,204 +0,0 @@
use std::sync::atomic::{AtomicUsize, Ordering};
use anyhow::{bail, Error};
use serde::{Deserialize, Serialize};
use proxmox::api::api;
use proxmox::api::schema::{ApiStringFormat, ApiType, Schema, StringSchema, ArraySchema, ReturnType};
use proxmox::const_regex;
use proxmox::sys::linux::procfs;
/// Unique Process/Task Identifier
///
/// We use this to uniquely identify worker task. UPIDs have a short
/// string repesentaion, which gives additional information about the
/// type of the task. for example:
/// ```text
/// UPID:{node}:{pid}:{pstart}:{task_id}:{starttime}:{worker_type}:{worker_id}:{userid}:
/// UPID:elsa:00004F37:0039E469:00000000:5CA78B83:garbage_collection::root@pam:
/// ```
/// Please note that we use tokio, so a single thread can run multiple
/// tasks.
// #[api] - manually implemented API type
#[derive(Debug, Clone)]
pub struct UPID {
/// The Unix PID
pub pid: libc::pid_t,
/// The Unix process start time from `/proc/pid/stat`
pub pstart: u64,
/// The task start time (Epoch)
pub starttime: i64,
/// The task ID (inside the process/thread)
pub task_id: usize,
/// Worker type (arbitrary ASCII string)
pub worker_type: String,
/// Worker ID (arbitrary ASCII string)
pub worker_id: Option<String>,
/// The authenticated entity who started the task
pub auth_id: String,
/// The node name.
pub node: String,
}
proxmox::forward_serialize_to_display!(UPID);
proxmox::forward_deserialize_to_from_str!(UPID);
const_regex! {
pub PROXMOX_UPID_REGEX = concat!(
r"^UPID:(?P<node>[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?):(?P<pid>[0-9A-Fa-f]{8}):",
r"(?P<pstart>[0-9A-Fa-f]{8,9}):(?P<task_id>[0-9A-Fa-f]{8,16}):(?P<starttime>[0-9A-Fa-f]{8}):",
r"(?P<wtype>[^:\s]+):(?P<wid>[^:\s]*):(?P<authid>[^:\s]+):$"
);
}
pub const PROXMOX_UPID_FORMAT: ApiStringFormat =
ApiStringFormat::Pattern(&PROXMOX_UPID_REGEX);
pub const UPID_SCHEMA: Schema = StringSchema::new("Unique Process/Task Identifier")
.min_length("UPID:N:12345678:12345678:12345678:::".len())
.max_length(128) // arbitrary
.format(&PROXMOX_UPID_FORMAT)
.schema();
impl ApiType for UPID {
const API_SCHEMA: Schema = UPID_SCHEMA;
}
impl UPID {
/// Create a new UPID
pub fn new(
worker_type: &str,
worker_id: Option<String>,
auth_id: String,
) -> Result<Self, Error> {
let pid = unsafe { libc::getpid() };
let bad: &[_] = &['/', ':', ' '];
if worker_type.contains(bad) {
bail!("illegal characters in worker type '{}'", worker_type);
}
if auth_id.contains(bad) {
bail!("illegal characters in auth_id '{}'", auth_id);
}
static WORKER_TASK_NEXT_ID: AtomicUsize = AtomicUsize::new(0);
let task_id = WORKER_TASK_NEXT_ID.fetch_add(1, Ordering::SeqCst);
Ok(UPID {
pid,
pstart: procfs::PidStat::read_from_pid(nix::unistd::Pid::from_raw(pid))?.starttime,
starttime: proxmox::tools::time::epoch_i64(),
task_id,
worker_type: worker_type.to_owned(),
worker_id,
auth_id,
node: proxmox::tools::nodename().to_owned(),
})
}
}
impl std::str::FromStr for UPID {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
if let Some(cap) = PROXMOX_UPID_REGEX.captures(s) {
let worker_id = if cap["wid"].is_empty() {
None
} else {
let wid = proxmox_systemd::unescape_unit(&cap["wid"])?;
Some(wid)
};
Ok(UPID {
pid: i32::from_str_radix(&cap["pid"], 16).unwrap(),
pstart: u64::from_str_radix(&cap["pstart"], 16).unwrap(),
starttime: i64::from_str_radix(&cap["starttime"], 16).unwrap(),
task_id: usize::from_str_radix(&cap["task_id"], 16).unwrap(),
worker_type: cap["wtype"].to_string(),
worker_id,
auth_id: cap["authid"].parse()?,
node: cap["node"].to_string(),
})
} else {
bail!("unable to parse UPID '{}'", s);
}
}
}
impl std::fmt::Display for UPID {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
let wid = if let Some(ref id) = self.worker_id {
proxmox_systemd::escape_unit(id, false)
} else {
String::new()
};
// Note: pstart can be > 32bit if uptime > 497 days, so this can result in
// more that 8 characters for pstart
write!(f, "UPID:{}:{:08X}:{:08X}:{:08X}:{:08X}:{}:{}:{}:",
self.node, self.pid, self.pstart, self.task_id, self.starttime, self.worker_type, wid, self.auth_id)
}
}
#[api()]
#[derive(Eq, PartialEq, Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum TaskStateType {
/// Ok
OK,
/// Warning
Warning,
/// Error
Error,
/// Unknown
Unknown,
}
#[api(
properties: {
upid: { schema: UPID::API_SCHEMA },
},
)]
#[derive(Serialize, Deserialize)]
/// Task properties.
pub struct TaskListItem {
pub upid: String,
/// The node name where the task is running on.
pub node: String,
/// The Unix PID
pub pid: i64,
/// The task start time (Epoch)
pub pstart: u64,
/// The task start time (Epoch)
pub starttime: i64,
/// Worker type (arbitrary ASCII string)
pub worker_type: String,
/// Worker ID (arbitrary ASCII string)
pub worker_id: Option<String>,
/// The authenticated entity who started the task
pub user: String,
/// The task end time (Epoch)
#[serde(skip_serializing_if="Option::is_none")]
pub endtime: Option<i64>,
/// Task end status
#[serde(skip_serializing_if="Option::is_none")]
pub status: Option<String>,
}
pub const NODE_TASKS_LIST_TASKS_RETURN_TYPE: ReturnType = ReturnType {
optional: false,
schema: &ArraySchema::new(
"A list of tasks.",
&TaskListItem::API_SCHEMA,
).schema(),
};

View File

@ -28,7 +28,7 @@ tower-service = "0.3.0"
xdg = "2.2"
pathpatterns = "0.1.2"
proxmox = { version = "0.13.3", default-features = false, features = [ "cli" ] }
proxmox = { version = "0.13.4", default-features = false, features = [ "cli" ] }
proxmox-fuse = "0.1.1"
proxmox-http = { version = "0.4.0", features = [ "client", "http-helpers", "websocket" ] }
pxar = { version = "0.10.1", features = [ "tokio-io" ] }

View File

@ -16,7 +16,7 @@ nix = "0.19.1"
regex = "1.2"
once_cell = "1.3.1"
proxmox = { version = "0.13.3", default-features = false, features = [ "cli" ] }
proxmox = { version = "0.13.4", default-features = false, features = [ "cli" ] }
pbs-api-types = { path = "../pbs-api-types" }
pbs-buildcfg = { path = "../pbs-buildcfg" }

View File

@ -23,7 +23,7 @@ zstd = { version = "0.6", features = [ "bindgen" ] }
pathpatterns = "0.1.2"
pxar = "0.10.1"
proxmox = { version = "0.13.3", default-features = false, features = [ "api-macro" ] }
proxmox = { version = "0.13.4", default-features = false, features = [ "api-macro" ] }
pbs-api-types = { path = "../pbs-api-types" }
pbs-tools = { path = "../pbs-tools" }

View File

@ -14,7 +14,7 @@ nix = "0.19.1"
regex = "1.2"
tokio = { version = "1.6", features = [] }
proxmox = "0.13.3"
proxmox = "0.13.4"
proxmox-fuse = "0.1.1"
pbs-tools = { path = "../pbs-tools" }

View File

@ -18,7 +18,7 @@ bitflags = "1.2.1"
regex = "1.2"
udev = ">= 0.3, <0.5"
proxmox = { version = "0.13.3", default-features = false, features = [] }
proxmox = { version = "0.13.4", default-features = false, features = [] }
pbs-api-types = { path = "../pbs-api-types" }
pbs-tools = { path = "../pbs-tools" }

View File

@ -30,7 +30,7 @@ url = "2.1"
walkdir = "2"
zstd = { version = "0.6", features = [ "bindgen" ] }
proxmox = { version = "0.13.3", default-features = false, features = [ "tokio" ] }
proxmox = { version = "0.13.4", default-features = false, features = [ "tokio" ] }
pbs-buildcfg = { path = "../pbs-buildcfg" }
pbs-runtime = { path = "../pbs-runtime" }

View File

@ -22,7 +22,7 @@ zstd = { version = "0.6", features = [ "bindgen" ] }
pathpatterns = "0.1.2"
pxar = { version = "0.10.1", features = [ "tokio-io" ] }
proxmox = { version = "0.13.3", features = [ "sortable-macro", "api-macro", "cli", "router" ] }
proxmox = { version = "0.13.4", features = [ "sortable-macro", "api-macro", "cli", "router" ] }
pbs-api-types = { path = "../pbs-api-types" }
pbs-buildcfg = { path = "../pbs-buildcfg" }
@ -31,5 +31,4 @@ pbs-client = { path = "../pbs-client" }
pbs-datastore = { path = "../pbs-datastore" }
pbs-fuse-loop = { path = "../pbs-fuse-loop" }
pbs-runtime = { path = "../pbs-runtime" }
proxmox-systemd = { path = "../proxmox-systemd" }
pbs-tools = { path = "../pbs-tools" }

View File

@ -118,7 +118,7 @@ fn complete_mapping_names<S: BuildHasher>(_arg: &str, _param: &HashMap<String, S
match pbs_fuse_loop::find_all_mappings() {
Ok(mappings) => mappings
.filter_map(|(name, _)| {
proxmox_systemd::unescape_unit(&name).ok()
proxmox::tools::systemd::unescape_unit(&name).ok()
}).collect(),
Err(_) => Vec::new()
}
@ -279,7 +279,7 @@ async fn mount_do(param: Value, pipe: Option<Fd>) -> Result<Value, Error> {
let reader = CachedChunkReader::new(chunk_reader, index, 8).seekable();
let name = &format!("{}:{}/{}", repo.to_string(), path, archive_name);
let name_escaped = proxmox_systemd::escape_unit(name, false);
let name_escaped = proxmox::tools::systemd::escape_unit(name, false);
let mut session = pbs_fuse_loop::FuseLoopSession::map_loop(size, reader, &name_escaped, options).await?;
let loopdev = session.loopdev_path.clone();
@ -341,7 +341,7 @@ fn unmap(
pbs_fuse_loop::cleanup_unused_run_files(None);
let mut any = false;
for (backing, loopdev) in pbs_fuse_loop::find_all_mappings()? {
let name = proxmox_systemd::unescape_unit(&backing)?;
let name = proxmox::tools::systemd::unescape_unit(&backing)?;
println!("{}:\t{}", loopdev.unwrap_or_else(|| "(unmapped)".to_string()), name);
any = true;
}
@ -360,7 +360,7 @@ fn unmap(
if name.starts_with("/dev/loop") {
pbs_fuse_loop::unmap_loopdev(name)?;
} else {
let name = proxmox_systemd::escape_unit(&name, false);
let name = proxmox::tools::systemd::escape_unit(&name, false);
pbs_fuse_loop::unmap_name(name)?;
}

View File

@ -16,7 +16,7 @@ tokio = { version = "1.6", features = [ "io-std", "rt", "rt-multi-thread", "time
pxar = { version = "0.10.1", features = [ "tokio-io" ] }
proxmox = { version = "0.13.3", features = [ "api-macro", "cli" ] }
proxmox = { version = "0.13.4", features = [ "api-macro", "cli" ] }
pbs-api-types = { path = "../pbs-api-types" }
pbs-buildcfg = { path = "../pbs-buildcfg" }
@ -24,5 +24,4 @@ pbs-config = { path = "../pbs-config" }
pbs-client = { path = "../pbs-client" }
pbs-datastore = { path = "../pbs-datastore" }
pbs-runtime = { path = "../pbs-runtime" }
proxmox-systemd = { path = "../proxmox-systemd" }
pbs-tools = { path = "../pbs-tools" }

View File

@ -80,7 +80,7 @@ impl VMStateMap {
fn make_name(repo: &BackupRepository, snap: &BackupDir) -> String {
let full = format!("qemu_{}/{}", repo, snap);
proxmox_systemd::escape_unit(&full, false)
proxmox::tools::systemd::escape_unit(&full, false)
}
/// remove non-responsive VMs from given map, returns 'true' if map was modified
@ -257,7 +257,7 @@ impl BlockRestoreDriver for QemuBlockDriver {
let resp = client
.get("api2/json/status", Some(json!({"keep-timeout": true})))
.await;
let name = proxmox_systemd::unescape_unit(n)
let name = proxmox::tools::systemd::unescape_unit(n)
.unwrap_or_else(|_| "<invalid name>".to_owned());
let mut extra = json!({"pid": s.pid, "cid": s.cid});
@ -295,7 +295,7 @@ impl BlockRestoreDriver for QemuBlockDriver {
fn stop(&self, id: String) -> Async<Result<(), Error>> {
async move {
let name = proxmox_systemd::escape_unit(&id, false);
let name = proxmox::tools::systemd::escape_unit(&id, false);
let mut map = VMStateMap::load()?;
let map_mod = cleanup_map(&mut map.map).await;
match map.map.get(&name) {
@ -325,7 +325,7 @@ impl BlockRestoreDriver for QemuBlockDriver {
match VMStateMap::load_read_only() {
Ok(state) => state
.iter()
.filter_map(|(name, _)| proxmox_systemd::unescape_unit(&name).ok())
.filter_map(|(name, _)| proxmox::tools::systemd::unescape_unit(&name).ok())
.collect(),
Err(_) => Vec::new(),
}

View File

@ -24,7 +24,7 @@ tokio-openssl = "0.6.1"
tower-service = "0.3.0"
url = "2.1"
proxmox = { version = "0.13.3", features = [ "router"] }
proxmox = { version = "0.13.4", features = [ "router"] }
# fixme: remove this dependency (pbs_tools::broadcast_future)
pbs-tools = { path = "../pbs-tools" }

View File

@ -26,7 +26,7 @@ tokio-util = { version = "0.6", features = [ "codec", "io" ] }
pathpatterns = "0.1.2"
pxar = { version = "0.10.1", features = [ "tokio-io" ] }
proxmox = { version = "0.13.3", features = [ "router", "sortable-macro" ] }
proxmox = { version = "0.13.4", features = [ "router", "sortable-macro" ] }
pbs-api-types = { path = "../pbs-api-types" }
pbs-runtime = { path = "../pbs-runtime" }

View File

@ -11,6 +11,6 @@ bitflags = "1.2.1"
lazy_static = "1.4"
nom = "5.1"
proxmox = { version = "0.13.3", default-features = false }
proxmox = { version = "0.13.4", default-features = false }
#pbs-tools = { path = "../pbs-tools" }

View File

@ -34,88 +34,6 @@ fn run_command(mut command: Command) -> Result<(), Error> {
Ok(())
}
/// Escape strings for usage in systemd unit names
pub fn escape_unit(mut unit: &str, is_path: bool) -> String {
if is_path {
unit = unit.trim_matches('/');
if unit.is_empty() {
return String::from("-");
}
}
let unit = unit.as_bytes();
let mut escaped = String::new();
for (i, c) in unit.iter().enumerate() {
if *c == b'/' {
escaped.push('-');
continue;
}
if (i == 0 && *c == b'.')
|| !(*c == b'_'
|| *c == b'.'
|| (*c >= b'0' && *c <= b'9')
|| (*c >= b'A' && *c <= b'Z')
|| (*c >= b'a' && *c <= b'z'))
{
escaped.push_str(&format!("\\x{:0x}", c));
} else {
escaped.push(*c as char);
}
}
escaped
}
fn parse_hex_digit(d: u8) -> Result<u8, Error> {
if d >= b'0' && d <= b'9' {
return Ok(d - b'0');
}
if d >= b'A' && d <= b'F' {
return Ok(d - b'A' + 10);
}
if d >= b'a' && d <= b'f' {
return Ok(d - b'a' + 10);
}
bail!("got invalid hex digit");
}
/// Unescape strings used in systemd unit names
pub fn unescape_unit(text: &str) -> Result<String, Error> {
let mut i = text.as_bytes();
let mut data: Vec<u8> = Vec::new();
loop {
if i.is_empty() {
break;
}
let next = i[0];
if next == b'\\' {
if i.len() < 4 {
bail!("short input");
}
if i[1] != b'x' {
bail!("unkwnown escape sequence");
}
let h1 = parse_hex_digit(i[2])?;
let h0 = parse_hex_digit(i[3])?;
data.push(h1 << 4 | h0);
i = &i[4..]
} else if next == b'-' {
data.push(b'/');
i = &i[1..]
} else {
data.push(next);
i = &i[1..]
}
}
let text = String::from_utf8(data)?;
Ok(text)
}
pub fn reload_daemon() -> Result<(), Error> {
let mut command = std::process::Command::new("systemctl");
command.arg("daemon-reload");
@ -178,6 +96,9 @@ pub fn reload_unit(unit: &str) -> Result<(), Error> {
#[test]
fn test_escape_unit() -> Result<(), Error> {
fn test_escape(i: &str, expected: &str, is_path: bool) {
use proxmox::tools::systemd::{escape_unit, unescape_unit};
let escaped = escape_unit(i, is_path);
assert_eq!(escaped, expected);
let unescaped = unescape_unit(&escaped).unwrap();

View File

@ -16,7 +16,7 @@ serde_json = "1.0"
tokio = { version = "1.6", features = [ "rt", "rt-multi-thread" ] }
pathpatterns = "0.1.2"
proxmox = { version = "0.13.3", default-features = false, features = [] }
proxmox = { version = "0.13.4", default-features = false, features = [] }
pxar = { version = "0.10.1", features = [ "tokio-io" ] }
pbs-client = { path = "../pbs-client" }

View File

@ -242,7 +242,7 @@ pub fn delete_datastore_disk(name: String) -> Result<(), Error> {
}
// disable systemd mount-unit
let mut mount_unit_name = proxmox_systemd::escape_unit(&path, true);
let mut mount_unit_name = proxmox::tools::systemd::escape_unit(&path, true);
mount_unit_name.push_str(".mount");
proxmox_systemd::disable_unit(&mount_unit_name)?;
@ -281,7 +281,7 @@ fn create_datastore_mount_unit(
what: &str,
) -> Result<String, Error> {
let mut mount_unit_name = proxmox_systemd::escape_unit(&mount_point, true);
let mut mount_unit_name = proxmox::tools::systemd::escape_unit(&mount_point, true);
mount_unit_name.push_str(".mount");
let mount_unit_path = format!("/etc/systemd/system/{}", mount_unit_name);

View File

@ -271,7 +271,7 @@ pub fn create_zpool(
worker.log(output);
if std::path::Path::new("/lib/systemd/system/zfs-import@.service").exists() {
let import_unit = format!("zfs-import@{}.service", proxmox_systemd::escape_unit(&name, false));
let import_unit = format!("zfs-import@{}.service", proxmox::tools::systemd::escape_unit(&name, false));
proxmox_systemd::enable_unit(&import_unit)?;
}

View File

@ -18,9 +18,9 @@ use once_cell::sync::OnceCell;
use proxmox::sys::linux::procfs;
use proxmox::try_block;
use proxmox::tools::fs::{create_path, replace_file, atomic_open_or_create_file, CreateOptions};
use proxmox::api::upid::UPID;
use pbs_tools::logrotate::{LogRotate, LogRotateFiles};
use pbs_api_types::UPID;
use proxmox_rest_server::{CommandoSocket, FileLogger, FileLogOptions};
struct TaskListLockGuard(File);

View File

@ -606,7 +606,7 @@ pub struct DeviceLockGuard(std::fs::File);
// Uses systemd escape_unit to compute a file name from `device_path`, the try
// to lock `/var/lock/<name>`.
fn open_device_lock(device_path: &str) -> Result<std::fs::File, Error> {
let lock_name = proxmox_systemd::escape_unit(device_path, true);
let lock_name = proxmox::tools::systemd::escape_unit(device_path, true);
let mut path = std::path::PathBuf::from(crate::tape::DRIVE_LOCK_DIR);
path.push(lock_name);