proxmox-backup/examples/test_chunk_size.rs

extern crate proxmox_backup;

// also see https://www.johndcook.com/blog/standard_deviation/

use anyhow::{Error};
use std::io::{Read, Write};

use proxmox_backup::backup::*;

struct ChunkWriter {
    chunker: Chunker,
    last_chunk: usize,
    chunk_offset: usize,

    chunk_count: usize,

    m_old: f64,
    m_new: f64,
    s_old: f64,
    s_new: f64,
}

impl ChunkWriter {

    fn new(chunk_size: usize) -> Self {
        ChunkWriter {
            chunker: Chunker::new(chunk_size),
            last_chunk: 0,
            chunk_offset: 0,
            chunk_count: 0,

            m_old: 0.0,
            m_new: 0.0,
            s_old: 0.0,
            s_new: 0.0,
        }
    }

    fn record_stat(&mut self, chunk_size: f64) {

        self.chunk_count += 1;

        if self.chunk_count == 1 {
            self.m_old = chunk_size;
            self.m_new = chunk_size;
            self.s_old = 0.0;
        } else {
            self.m_new = self.m_old + (chunk_size - self.m_old)/(self.chunk_count as f64);
            self.s_new = self.s_old +
                (chunk_size - self.m_old)*(chunk_size - self.m_new);
            // set up for next iteration
            self.m_old = self.m_new;
            self.s_old = self.s_new;
        }

        let variance = if self.chunk_count > 1 {
            self.s_new/((self.chunk_count -1)as f64)
        } else { 0.0 };

        let std_deviation = variance.sqrt();
        let deviation_per = (std_deviation*100.0)/self.m_new;
        println!("COUNT {:10} SIZE {:10} MEAN {:10} DEVIATION {:3}%", self.chunk_count, chunk_size, self.m_new as usize, deviation_per as usize);
    }
}

impl Write for ChunkWriter {

    fn write(&mut self, data: &[u8]) -> std::result::Result<usize, std::io::Error> {

        let chunker = &mut self.chunker;

        let pos = chunker.scan(data);

        if pos > 0 {
            self.chunk_offset += pos;

            let chunk_size = self.chunk_offset - self.last_chunk;

            self.record_stat(chunk_size as f64);

            self.last_chunk = self.chunk_offset;
            Ok(pos)

        } else {
            self.chunk_offset += data.len();
            Ok(data.len())
        }
    }

    fn flush(&mut self) -> std::result::Result<(), std::io::Error> {
        Ok(())
    }
}

fn main() -> Result<(), Error> {

    let mut file = std::fs::File::open("/dev/urandom")?;

    let mut bytes = 0;

    let mut buffer = [0u8; 64*1024];

    let mut writer = ChunkWriter::new(4096*1024);

    loop {

        file.read_exact(&mut buffer)?;
        bytes += buffer.len();

        writer.write_all(&buffer)?;

        if bytes > 1024*1024*1024 { break; }
    }

    Ok(())
}