api/tools: split out apt helpers from api to own module

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
This commit is contained in:
Thomas Lamprecht 2020-10-31 20:40:05 +01:00
parent 5911f74096
commit e6513bd5de
3 changed files with 299 additions and 291 deletions

View File

@ -1,302 +1,16 @@
use std::collections::HashSet;
use apt_pkg_native::Cache;
use anyhow::{Error, bail, format_err}; use anyhow::{Error, bail, format_err};
use serde_json::{json, Value}; use serde_json::{json, Value};
use proxmox::{list_subdirs_api_method, const_regex}; use proxmox::list_subdirs_api_method;
use proxmox::api::{api, RpcEnvironment, RpcEnvironmentType, Permission}; use proxmox::api::{api, RpcEnvironment, RpcEnvironmentType, Permission};
use proxmox::api::router::{Router, SubdirMap}; use proxmox::api::router::{Router, SubdirMap};
use crate::server::WorkerTask; use crate::server::WorkerTask;
use crate::tools::http; use crate::tools::{apt, http};
use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY}; use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
use crate::api2::types::{Authid, APTUpdateInfo, NODE_SCHEMA, UPID_SCHEMA}; use crate::api2::types::{Authid, APTUpdateInfo, NODE_SCHEMA, UPID_SCHEMA};
const_regex! {
VERSION_EPOCH_REGEX = r"^\d+:";
FILENAME_EXTRACT_REGEX = r"^.*/.*?_(.*)_Packages$";
}
// FIXME: once the 'changelog' API call switches over to 'apt-get changelog' only,
// consider removing this function entirely, as it's value is never used anywhere
// then (widget-toolkit doesn't use the value either)
fn get_changelog_url(
package: &str,
filename: &str,
version: &str,
origin: &str,
component: &str,
) -> Result<String, Error> {
if origin == "" {
bail!("no origin available for package {}", package);
}
if origin == "Debian" {
let mut command = std::process::Command::new("apt-get");
command.arg("changelog");
command.arg("--print-uris");
command.arg(package);
let output = crate::tools::run_command(command, None)?; // format: 'http://foo/bar' package.changelog
let output = match output.splitn(2, ' ').next() {
Some(output) => {
if output.len() < 2 {
bail!("invalid output (URI part too short) from 'apt-get changelog --print-uris': {}", output)
}
output[1..output.len()-1].to_owned()
},
None => bail!("invalid output from 'apt-get changelog --print-uris': {}", output)
};
return Ok(output);
} else if origin == "Proxmox" {
// FIXME: Use above call to 'apt changelog <pkg> --print-uris' as well.
// Currently not possible as our packages do not have a URI set in their Release file.
let version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(version, "");
let base = match (FILENAME_EXTRACT_REGEX.regex_obj)().captures(filename) {
Some(captures) => {
let base_capture = captures.get(1);
match base_capture {
Some(base_underscore) => base_underscore.as_str().replace("_", "/"),
None => bail!("incompatible filename, cannot find regex group")
}
},
None => bail!("incompatible filename, doesn't match regex")
};
return Ok(format!("http://download.proxmox.com/{}/{}_{}.changelog",
base, package, version));
}
bail!("unknown origin ({}) or component ({})", origin, component)
}
struct FilterData<'a> {
// this is version info returned by APT
installed_version: Option<&'a str>,
candidate_version: &'a str,
// this is the version info the filter is supposed to check
active_version: &'a str,
}
enum PackagePreSelect {
OnlyInstalled,
OnlyNew,
All,
}
fn list_installed_apt_packages<F: Fn(FilterData) -> bool>(
filter: F,
only_versions_for: Option<&str>,
) -> Vec<APTUpdateInfo> {
let mut ret = Vec::new();
let mut depends = HashSet::new();
// note: this is not an 'apt update', it just re-reads the cache from disk
let mut cache = Cache::get_singleton();
cache.reload();
let mut cache_iter = match only_versions_for {
Some(name) => cache.find_by_name(name),
None => cache.iter()
};
loop {
match cache_iter.next() {
Some(view) => {
let di = if only_versions_for.is_some() {
query_detailed_info(
PackagePreSelect::All,
&filter,
view,
None
)
} else {
query_detailed_info(
PackagePreSelect::OnlyInstalled,
&filter,
view,
Some(&mut depends)
)
};
if let Some(info) = di {
ret.push(info);
}
if only_versions_for.is_some() {
break;
}
},
None => {
drop(cache_iter);
// also loop through missing dependencies, as they would be installed
for pkg in depends.iter() {
let mut iter = cache.find_by_name(&pkg);
let view = match iter.next() {
Some(view) => view,
None => continue // package not found, ignore
};
let di = query_detailed_info(
PackagePreSelect::OnlyNew,
&filter,
view,
None
);
if let Some(info) = di {
ret.push(info);
}
}
break;
}
}
}
return ret;
}
fn query_detailed_info<'a, F, V>(
pre_select: PackagePreSelect,
filter: F,
view: V,
depends: Option<&mut HashSet<String>>,
) -> Option<APTUpdateInfo>
where
F: Fn(FilterData) -> bool,
V: std::ops::Deref<Target = apt_pkg_native::sane::PkgView<'a>>
{
let current_version = view.current_version();
let candidate_version = view.candidate_version();
let (current_version, candidate_version) = match pre_select {
PackagePreSelect::OnlyInstalled => match (current_version, candidate_version) {
(Some(cur), Some(can)) => (Some(cur), can), // package installed and there is an update
(Some(cur), None) => (Some(cur.clone()), cur), // package installed and up-to-date
(None, Some(_)) => return None, // package could be installed
(None, None) => return None, // broken
},
PackagePreSelect::OnlyNew => match (current_version, candidate_version) {
(Some(_), Some(_)) => return None,
(Some(_), None) => return None,
(None, Some(can)) => (None, can),
(None, None) => return None,
},
PackagePreSelect::All => match (current_version, candidate_version) {
(Some(cur), Some(can)) => (Some(cur), can),
(Some(cur), None) => (Some(cur.clone()), cur),
(None, Some(can)) => (None, can),
(None, None) => return None,
},
};
// get additional information via nested APT 'iterators'
let mut view_iter = view.versions();
while let Some(ver) = view_iter.next() {
let package = view.name();
let version = ver.version();
let mut origin_res = "unknown".to_owned();
let mut section_res = "unknown".to_owned();
let mut priority_res = "unknown".to_owned();
let mut change_log_url = "".to_owned();
let mut short_desc = package.clone();
let mut long_desc = "".to_owned();
let fd = FilterData {
installed_version: current_version.as_deref(),
candidate_version: &candidate_version,
active_version: &version,
};
if filter(fd) {
if let Some(section) = ver.section() {
section_res = section;
}
if let Some(prio) = ver.priority_type() {
priority_res = prio;
}
// assume every package has only one origin file (not
// origin, but origin *file*, for some reason those seem to
// be different concepts in APT)
let mut origin_iter = ver.origin_iter();
let origin = origin_iter.next();
if let Some(origin) = origin {
if let Some(sd) = origin.short_desc() {
short_desc = sd;
}
if let Some(ld) = origin.long_desc() {
long_desc = ld;
}
// the package files appear in priority order, meaning
// the one for the candidate version is first - this is fine
// however, as the source package should be the same for all
// versions anyway
let mut pkg_iter = origin.file();
let pkg_file = pkg_iter.next();
if let Some(pkg_file) = pkg_file {
if let Some(origin_name) = pkg_file.origin() {
origin_res = origin_name;
}
let filename = pkg_file.file_name();
let component = pkg_file.component();
// build changelog URL from gathered information
// ignore errors, use empty changelog instead
let url = get_changelog_url(&package, &filename,
&version, &origin_res, &component);
if let Ok(url) = url {
change_log_url = url;
}
}
}
if let Some(depends) = depends {
let mut dep_iter = ver.dep_iter();
loop {
let dep = match dep_iter.next() {
Some(dep) if dep.dep_type() != "Depends" => continue,
Some(dep) => dep,
None => break
};
let dep_pkg = dep.target_pkg();
let name = dep_pkg.name();
depends.insert(name);
}
}
return Some(APTUpdateInfo {
package,
title: short_desc,
arch: view.arch(),
description: long_desc,
change_log_url,
origin: origin_res,
version: candidate_version.clone(),
old_version: match current_version {
Some(vers) => vers,
None => "".to_owned()
},
priority: priority_res,
section: section_res,
});
}
}
return None;
}
#[api( #[api(
input: { input: {
properties: { properties: {
@ -308,7 +22,9 @@ where
returns: { returns: {
description: "A list of packages with available updates.", description: "A list of packages with available updates.",
type: Array, type: Array,
items: { type: APTUpdateInfo }, items: {
type: APTUpdateInfo
},
}, },
access: { access: {
permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false), permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
@ -316,7 +32,7 @@ where
)] )]
/// List available APT updates /// List available APT updates
fn apt_update_available(_param: Value) -> Result<Value, Error> { fn apt_update_available(_param: Value) -> Result<Value, Error> {
let all_upgradeable = list_installed_apt_packages(|data| { let all_upgradeable = apt::list_installed_apt_packages(|data| {
data.candidate_version == data.active_version && data.candidate_version == data.active_version &&
data.installed_version != Some(data.candidate_version) data.installed_version != Some(data.candidate_version)
}, None); }, None);
@ -406,7 +122,7 @@ fn apt_get_changelog(
let name = crate::tools::required_string_param(&param, "name")?.to_owned(); let name = crate::tools::required_string_param(&param, "name")?.to_owned();
let version = param["version"].as_str(); let version = param["version"].as_str();
let pkg_info = list_installed_apt_packages(|data| { let pkg_info = apt::list_installed_apt_packages(|data| {
match version { match version {
Some(version) => version == data.active_version, Some(version) => version == data.active_version,
None => data.active_version == data.candidate_version None => data.active_version == data.candidate_version

View File

@ -19,6 +19,7 @@ use proxmox::tools::vec;
pub use proxmox::tools::fd::Fd; pub use proxmox::tools::fd::Fd;
pub mod acl; pub mod acl;
pub mod apt;
pub mod async_io; pub mod async_io;
pub mod borrow; pub mod borrow;
pub mod cert; pub mod cert;

291
src/tools/apt.rs Normal file
View File

@ -0,0 +1,291 @@
use std::collections::HashSet;
use anyhow::{Error, bail};
use apt_pkg_native::Cache;
use proxmox::const_regex;
use crate::api2::types::APTUpdateInfo;
const_regex! {
VERSION_EPOCH_REGEX = r"^\d+:";
FILENAME_EXTRACT_REGEX = r"^.*/.*?_(.*)_Packages$";
}
// FIXME: once the 'changelog' API call switches over to 'apt-get changelog' only,
// consider removing this function entirely, as it's value is never used anywhere
// then (widget-toolkit doesn't use the value either)
fn get_changelog_url(
package: &str,
filename: &str,
version: &str,
origin: &str,
component: &str,
) -> Result<String, Error> {
if origin == "" {
bail!("no origin available for package {}", package);
}
if origin == "Debian" {
let mut command = std::process::Command::new("apt-get");
command.arg("changelog");
command.arg("--print-uris");
command.arg(package);
let output = crate::tools::run_command(command, None)?; // format: 'http://foo/bar' package.changelog
let output = match output.splitn(2, ' ').next() {
Some(output) => {
if output.len() < 2 {
bail!("invalid output (URI part too short) from 'apt-get changelog --print-uris': {}", output)
}
output[1..output.len()-1].to_owned()
},
None => bail!("invalid output from 'apt-get changelog --print-uris': {}", output)
};
return Ok(output);
} else if origin == "Proxmox" {
// FIXME: Use above call to 'apt changelog <pkg> --print-uris' as well.
// Currently not possible as our packages do not have a URI set in their Release file.
let version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(version, "");
let base = match (FILENAME_EXTRACT_REGEX.regex_obj)().captures(filename) {
Some(captures) => {
let base_capture = captures.get(1);
match base_capture {
Some(base_underscore) => base_underscore.as_str().replace("_", "/"),
None => bail!("incompatible filename, cannot find regex group")
}
},
None => bail!("incompatible filename, doesn't match regex")
};
return Ok(format!("http://download.proxmox.com/{}/{}_{}.changelog",
base, package, version));
}
bail!("unknown origin ({}) or component ({})", origin, component)
}
pub struct FilterData<'a> {
// this is version info returned by APT
pub installed_version: Option<&'a str>,
pub candidate_version: &'a str,
// this is the version info the filter is supposed to check
pub active_version: &'a str,
}
enum PackagePreSelect {
OnlyInstalled,
OnlyNew,
All,
}
pub fn list_installed_apt_packages<F: Fn(FilterData) -> bool>(
filter: F,
only_versions_for: Option<&str>,
) -> Vec<APTUpdateInfo> {
let mut ret = Vec::new();
let mut depends = HashSet::new();
// note: this is not an 'apt update', it just re-reads the cache from disk
let mut cache = Cache::get_singleton();
cache.reload();
let mut cache_iter = match only_versions_for {
Some(name) => cache.find_by_name(name),
None => cache.iter()
};
loop {
match cache_iter.next() {
Some(view) => {
let di = if only_versions_for.is_some() {
query_detailed_info(
PackagePreSelect::All,
&filter,
view,
None
)
} else {
query_detailed_info(
PackagePreSelect::OnlyInstalled,
&filter,
view,
Some(&mut depends)
)
};
if let Some(info) = di {
ret.push(info);
}
if only_versions_for.is_some() {
break;
}
},
None => {
drop(cache_iter);
// also loop through missing dependencies, as they would be installed
for pkg in depends.iter() {
let mut iter = cache.find_by_name(&pkg);
let view = match iter.next() {
Some(view) => view,
None => continue // package not found, ignore
};
let di = query_detailed_info(
PackagePreSelect::OnlyNew,
&filter,
view,
None
);
if let Some(info) = di {
ret.push(info);
}
}
break;
}
}
}
return ret;
}
fn query_detailed_info<'a, F, V>(
pre_select: PackagePreSelect,
filter: F,
view: V,
depends: Option<&mut HashSet<String>>,
) -> Option<APTUpdateInfo>
where
F: Fn(FilterData) -> bool,
V: std::ops::Deref<Target = apt_pkg_native::sane::PkgView<'a>>
{
let current_version = view.current_version();
let candidate_version = view.candidate_version();
let (current_version, candidate_version) = match pre_select {
PackagePreSelect::OnlyInstalled => match (current_version, candidate_version) {
(Some(cur), Some(can)) => (Some(cur), can), // package installed and there is an update
(Some(cur), None) => (Some(cur.clone()), cur), // package installed and up-to-date
(None, Some(_)) => return None, // package could be installed
(None, None) => return None, // broken
},
PackagePreSelect::OnlyNew => match (current_version, candidate_version) {
(Some(_), Some(_)) => return None,
(Some(_), None) => return None,
(None, Some(can)) => (None, can),
(None, None) => return None,
},
PackagePreSelect::All => match (current_version, candidate_version) {
(Some(cur), Some(can)) => (Some(cur), can),
(Some(cur), None) => (Some(cur.clone()), cur),
(None, Some(can)) => (None, can),
(None, None) => return None,
},
};
// get additional information via nested APT 'iterators'
let mut view_iter = view.versions();
while let Some(ver) = view_iter.next() {
let package = view.name();
let version = ver.version();
let mut origin_res = "unknown".to_owned();
let mut section_res = "unknown".to_owned();
let mut priority_res = "unknown".to_owned();
let mut change_log_url = "".to_owned();
let mut short_desc = package.clone();
let mut long_desc = "".to_owned();
let fd = FilterData {
installed_version: current_version.as_deref(),
candidate_version: &candidate_version,
active_version: &version,
};
if filter(fd) {
if let Some(section) = ver.section() {
section_res = section;
}
if let Some(prio) = ver.priority_type() {
priority_res = prio;
}
// assume every package has only one origin file (not
// origin, but origin *file*, for some reason those seem to
// be different concepts in APT)
let mut origin_iter = ver.origin_iter();
let origin = origin_iter.next();
if let Some(origin) = origin {
if let Some(sd) = origin.short_desc() {
short_desc = sd;
}
if let Some(ld) = origin.long_desc() {
long_desc = ld;
}
// the package files appear in priority order, meaning
// the one for the candidate version is first - this is fine
// however, as the source package should be the same for all
// versions anyway
let mut pkg_iter = origin.file();
let pkg_file = pkg_iter.next();
if let Some(pkg_file) = pkg_file {
if let Some(origin_name) = pkg_file.origin() {
origin_res = origin_name;
}
let filename = pkg_file.file_name();
let component = pkg_file.component();
// build changelog URL from gathered information
// ignore errors, use empty changelog instead
let url = get_changelog_url(&package, &filename,
&version, &origin_res, &component);
if let Ok(url) = url {
change_log_url = url;
}
}
}
if let Some(depends) = depends {
let mut dep_iter = ver.dep_iter();
loop {
let dep = match dep_iter.next() {
Some(dep) if dep.dep_type() != "Depends" => continue,
Some(dep) => dep,
None => break
};
let dep_pkg = dep.target_pkg();
let name = dep_pkg.name();
depends.insert(name);
}
}
return Some(APTUpdateInfo {
package,
title: short_desc,
arch: view.arch(),
description: long_desc,
change_log_url,
origin: origin_res,
version: candidate_version.clone(),
old_version: match current_version {
Some(vers) => vers,
None => "".to_owned()
},
priority: priority_res,
section: section_res,
});
}
}
return None;
}