proxmox-rrd: implement new CBOR based format

Storing much more data points now got get better graphs.

Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

src/api2/node/rrd.rs

@@ -1,5 +1,6 @@
-use anyhow::Error;
+use anyhow::{bail, Error};
 use serde_json::{Value, json};
+use std::collections::BTreeMap;
 
 use proxmox_router::{Permission, Router};
 use proxmox_schema::api;
@@ -8,8 +9,6 @@ use pbs_api_types::{
     NODE_SCHEMA, RRDMode, RRDTimeFrameResolution, PRIV_SYS_AUDIT,
 };
 
-use proxmox_rrd::rrd::RRD_DATA_ENTRIES;
-
 use crate::get_rrd_cache;
 
 pub fn create_value_from_rrd(
@@ -19,32 +18,44 @@ pub fn create_value_from_rrd(
     cf: RRDMode,
 ) -> Result<Value, Error> {
 
-    let mut result = Vec::new();
+    let mut result: Vec<Value> = Vec::new();
     let now = proxmox_time::epoch_f64();
 
     let rrd_cache = get_rrd_cache()?;
 
+    let mut timemap = BTreeMap::new();
+
+    let mut last_resolution = None;
+
     for name in list {
-        let (start, reso, list) = match rrd_cache.extract_cached_data(basedir, name, now, timeframe, cf) {
+        let (start, reso, data) = match rrd_cache.extract_cached_data(basedir, name, now, timeframe, cf)? {
             Some(result) => result,
             None => continue,
         };
 
+        if let Some(expected_resolution) = last_resolution  {
+            if reso != expected_resolution {
+                bail!("got unexpected RRD resolution ({} != {})", reso, expected_resolution);
+            }
+        } else {
+            last_resolution = Some(reso);
+        }
+
         let mut t = start;
-        for index in 0..RRD_DATA_ENTRIES {
-            if result.len() <= index {
-                if let Some(value) = list[index] {
-                    result.push(json!({ "time": t, *name: value }));
-                } else {
-                    result.push(json!({ "time": t }));
-                }
-            } else if let Some(value) = list[index] {
-                result[index][name] = value.into();
+
+        for index in 0..data.len() {
+            let entry = timemap.entry(t).or_insert(json!({ "time": t }));
+            if let Some(value) = data[index] {
+                entry[*name] = value.into();
             }
             t += reso;
         }
     }
 
+    for item in timemap.values() {
+        result.push(item.clone());
+    }
+
     Ok(result.into())
 }
 

src/api2/status.rs

@@ -134,8 +134,8 @@ pub fn datastore_status(
             RRDMode::Average,
         );
 
-        let total_res = get_rrd("total");
-        let used_res = get_rrd("used");
+        let total_res = get_rrd("total")?;
+        let used_res = get_rrd("used")?;
 
         if let (Some((start, reso, total_list)), Some((_, _, used_list))) = (total_res, used_res) {
             let mut usage_list: Vec<f64> = Vec::new();

src/bin/proxmox-backup-proxy.rs

@@ -24,7 +24,7 @@ use proxmox_router::{RpcEnvironment, RpcEnvironmentType, UserInformation};
 
 use pbs_tools::{task_log, task_warn};
 use pbs_datastore::DataStore;
-use proxmox_rrd::DST;
+use proxmox_rrd::rrd::DST;
 
 use proxmox_rest_server::{
     rotate_task_log_archive, extract_cookie , AuthError, ApiConfig, RestServer, RestEnvironment,