server: implement access log rotation with re-open via command socket

re-use the future we already have for task log rotation to trigger
it.

Move the FileLogger in ApiConfig into an Arc, so that we can actually
update it and REST using the new one.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

src/bin/proxmox-backup-api.rs

@@ -54,7 +54,7 @@ async fn run() -> Result<(), Error> {
 
     let mut commando_sock = server::CommandoSocket::new(server::our_ctrl_sock());
 
-    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN)?;
+    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN, &mut commando_sock)?;
 
     let rest_server = RestServer::new(config);
 

src/bin/proxmox-backup-proxy.rs

@@ -41,6 +41,7 @@ use proxmox_backup::tools::{
         DiskManage,
         zfs_pool_stats,
     },
+    logrotate::LogRotate,
     socket::{
         set_tcp_keepalive,
         PROXMOX_BACKUP_TCP_KEEPALIVE_TIME,
@@ -96,7 +97,7 @@ async fn run() -> Result<(), Error> {
 
     let mut commando_sock = server::CommandoSocket::new(server::our_ctrl_sock());
 
-    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN)?;
+    config.enable_file_log(buildcfg::API_ACCESS_LOG_FN, &mut commando_sock)?;
 
     let rest_server = RestServer::new(config);
 
@@ -531,6 +532,21 @@ async fn schedule_task_log_rotate() {
                     worker.log(format!("task log archive was not rotated"));
                 }
 
+                let max_size = 32 * 1024 * 1024 - 1;
+                let max_files = 14;
+                let mut logrotate = LogRotate::new(buildcfg::API_ACCESS_LOG_FN, true)
+                        .ok_or_else(|| format_err!("could not get API access log file names"))?;
+
+                let has_rotated = logrotate.rotate(max_size, None, Some(max_files))?;
+                if has_rotated {
+                    println!("rotated access log, telling daemons to re-open log file");
+                    proxmox_backup::tools::runtime::block_on(command_reopen_logfiles())?;
+
+                    worker.log(format!("API access log was rotated"));
+                } else {
+                    worker.log(format!("API access log was not rotated"));
+                }
+
                 Ok(())
             });
 
@@ -548,6 +564,22 @@ async fn schedule_task_log_rotate() {
 
 }
 
+async fn command_reopen_logfiles() -> Result<(), Error> {
+    // only care about the most recent daemon instance for each, proxy & api, as other older ones
+    // should not respond to new requests anyway, but only finish their current one and then exit.
+    let sock = server::our_ctrl_sock();
+    server::send_command(sock, serde_json::json!({
+        "command": "api-access-log-reopen",
+    })).await?;
+
+    let pid = server::read_pid(buildcfg::PROXMOX_BACKUP_API_PID_FN)?;
+    let sock = server::ctrl_sock_from_pid(pid);
+    server::send_command(sock, serde_json::json!({
+        "command": "api-access-log-reopen",
+    })).await?;
+    Ok(())
+}
+
 async fn run_stat_generator() {
 
     let mut count = 0;

src/server/config.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 use std::path::PathBuf;
 use std::time::SystemTime;
 use std::fs::metadata;
-use std::sync::{Mutex, RwLock};
+use std::sync::{Arc, Mutex, RwLock};
 
 use anyhow::{bail, Error, format_err};
 use hyper::Method;
@@ -21,7 +21,7 @@ pub struct ApiConfig {
     env_type: RpcEnvironmentType,
     templates: RwLock<Handlebars<'static>>,
     template_files: RwLock<HashMap<String, (SystemTime, PathBuf)>>,
-    request_log: Option<Mutex<FileLogger>>,
+    request_log: Option<Arc<Mutex<FileLogger>>>,
 }
 
 impl ApiConfig {
@@ -124,7 +124,11 @@ impl ApiConfig {
         }
     }
 
-    pub fn enable_file_log<P>(&mut self, path: P) -> Result<(), Error>
+    pub fn enable_file_log<P>(
+        &mut self,
+        path: P,
+        commando_sock: &mut super::CommandoSocket,
+    ) -> Result<(), Error>
     where
         P: Into<PathBuf>
     {
@@ -142,11 +146,19 @@ impl ApiConfig {
             owned_by_backup: true,
             ..Default::default()
         };
-        self.request_log = Some(Mutex::new(FileLogger::new(&path, logger_options)?));
+        let request_log = Arc::new(Mutex::new(FileLogger::new(&path, logger_options)?));
+        self.request_log = Some(Arc::clone(&request_log));
+
+        commando_sock.register_command("api-access-log-reopen".into(), move |_args| {
+            println!("re-opening log file");
+            request_log.lock().unwrap().reopen()?;
+            Ok(serde_json::Value::Null)
+        })?;
 
         Ok(())
     }
-    pub fn get_file_log(&self) -> Option<&Mutex<FileLogger>> {
+
+    pub fn get_file_log(&self) -> Option<&Arc<Mutex<FileLogger>>> {
         self.request_log.as_ref()
     }
 }

src/server/rest.rs

@@ -112,7 +112,7 @@ pub struct ApiService {
 }
 
 fn log_response(
-    logfile: Option<&Mutex<FileLogger>>,
+    logfile: Option<&Arc<Mutex<FileLogger>>>,
     peer: &std::net::SocketAddr,
     method: hyper::Method,
     path_query: &str,