backup-client: read file by chunks

This commit is contained in:
root 2018-12-14 13:39:41 +01:00
parent ff5d37074d
commit a914a77448

View File

@ -3,6 +3,11 @@ extern crate apitest;
use failure::*;
use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::io::ErrorKind;
use std::io::prelude::*;
use std::iter::Iterator;
use apitest::cli::command::*;
use apitest::api::schema::*;
@ -13,10 +18,92 @@ use std::path::{Path, PathBuf};
use apitest::config::datastore;
fn required_string_param<'a>(param: &'a Value, name: &str) -> &'a str {
param[name].as_str().expect(&format!("missing parameter '{}'", name))
}
fn file_chunker<C>(
mut file: File,
chunk_cb: C
) -> Result<(), Error>
where C: Fn(usize, &[u8]) -> Result<bool, Error>
{
let chunk_size = 64*1024;
let mut buf = vec![0u8; 4*1024*1024];
let mut pos = 0;
let mut file_pos = 0;
loop {
let mut eof = false;
let mut tmp = &mut buf[..];
// try to read large portions, at least chunk_size
while pos < chunk_size {
match file.read(tmp) {
Ok(0) => { eof = true; break; },
Ok(n) => {
pos += n;
if pos > chunk_size { break; }
tmp = &mut tmp[n..];
}
Err(ref e) if e.kind() == ErrorKind::Interrupted => { /* try again */ }
Err(e) => bail!("read error - {}", e.to_string()),
}
}
println!("READ {} {}", pos, eof);
let mut start = 0;
while start + chunk_size <= pos {
if !(chunk_cb)(file_pos, &buf[start..start+chunk_size])? { break; }
file_pos += chunk_size;
start += chunk_size;
}
if eof {
if start < pos {
(chunk_cb)(file_pos, &buf[start..pos])?;
//file_pos += pos - start;
}
break;
} else {
let rest = pos - start;
if rest > 0 {
let ptr = buf.as_mut_ptr();
unsafe { std::ptr::copy_nonoverlapping(ptr.add(start), ptr, rest); }
pos = rest;
} else {
pos = 0;
}
}
}
Ok(())
}
fn backup_file(param: Value, _info: &ApiMethod) -> Result<Value, Error> {
println!("Backup file '{}'", param["filename"].as_str().unwrap());
let filename = required_string_param(&param, "filename");
let store = required_string_param(&param, "store");
let config = datastore::config()?;
let (_, store_config) = config.sections.get(store)
.ok_or(format_err!("no such datastore '{}'", store))?;
let path = store_config["path"].as_str().unwrap();
let _store = ChunkStore::open(path)?;
println!("Backup file '{}' to '{}'", filename, store);
let file = std::fs::File::open(filename)?;
file_chunker(file, |pos, chunk| {
println!("CHUNK {} {}", pos, chunk.len());
Ok(true)
})?;
Ok(Value::Null)
}
@ -31,7 +118,7 @@ fn main() {
.required("store", StringSchema::new("Datastore name."))
))
.arg_param(vec!["filename"]);
if let Err(err) = run_cli_command(&cmd_def.into()) {
eprintln!("Error: {}", err);
print_cli_usage();