2018-12-17 12:00:39 +00:00
|
|
|
use failure::*;
|
|
|
|
|
2018-12-18 10:06:03 +00:00
|
|
|
use std::path::{PathBuf, Path};
|
2018-12-22 16:37:25 +00:00
|
|
|
use std::collections::HashMap;
|
|
|
|
use lazy_static::lazy_static;
|
|
|
|
use std::sync::{Mutex, Arc};
|
2018-12-17 12:00:39 +00:00
|
|
|
|
|
|
|
use crate::config::datastore;
|
|
|
|
use super::chunk_store::*;
|
|
|
|
use super::image_index::*;
|
2018-12-31 16:30:08 +00:00
|
|
|
use super::archive_index::*;
|
2018-12-17 12:00:39 +00:00
|
|
|
|
2019-01-18 11:01:37 +00:00
|
|
|
use chrono::{Utc, TimeZone};
|
|
|
|
|
2018-12-17 12:00:39 +00:00
|
|
|
pub struct DataStore {
|
2019-01-15 10:38:26 +00:00
|
|
|
chunk_store: Arc<ChunkStore>,
|
2018-12-22 15:58:16 +00:00
|
|
|
gc_mutex: Mutex<bool>,
|
2018-12-17 12:00:39 +00:00
|
|
|
}
|
|
|
|
|
2018-12-22 16:37:25 +00:00
|
|
|
lazy_static!{
|
|
|
|
static ref datastore_map: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new());
|
|
|
|
}
|
|
|
|
|
2018-12-17 12:00:39 +00:00
|
|
|
impl DataStore {
|
|
|
|
|
2018-12-22 16:37:25 +00:00
|
|
|
pub fn lookup_datastore(name: &str) -> Result<Arc<DataStore>, Error> {
|
|
|
|
|
|
|
|
let config = datastore::config()?;
|
|
|
|
let (_, store_config) = config.sections.get(name)
|
|
|
|
.ok_or(format_err!("no such datastore '{}'", name))?;
|
|
|
|
|
|
|
|
let path = store_config["path"].as_str().unwrap();
|
|
|
|
|
|
|
|
let mut map = datastore_map.lock().unwrap();
|
|
|
|
|
|
|
|
if let Some(datastore) = map.get(name) {
|
|
|
|
// Compare Config - if changed, create new Datastore object!
|
2018-12-25 12:29:27 +00:00
|
|
|
if datastore.chunk_store.base == PathBuf::from(path) {
|
2018-12-22 16:37:25 +00:00
|
|
|
return Ok(datastore.clone());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if let Ok(datastore) = DataStore::open(name) {
|
|
|
|
let datastore = Arc::new(datastore);
|
|
|
|
map.insert(name.to_string(), datastore.clone());
|
|
|
|
return Ok(datastore);
|
|
|
|
}
|
|
|
|
|
|
|
|
bail!("store not found");
|
|
|
|
}
|
|
|
|
|
2018-12-17 12:00:39 +00:00
|
|
|
pub fn open(store_name: &str) -> Result<Self, Error> {
|
|
|
|
|
|
|
|
let config = datastore::config()?;
|
|
|
|
let (_, store_config) = config.sections.get(store_name)
|
|
|
|
.ok_or(format_err!("no such datastore '{}'", store_name))?;
|
|
|
|
|
|
|
|
let path = store_config["path"].as_str().unwrap();
|
|
|
|
|
2018-12-19 12:40:26 +00:00
|
|
|
let chunk_store = ChunkStore::open(store_name, path)?;
|
2018-12-17 12:00:39 +00:00
|
|
|
|
|
|
|
Ok(Self {
|
2019-01-15 10:38:26 +00:00
|
|
|
chunk_store: Arc::new(chunk_store),
|
2018-12-22 15:58:16 +00:00
|
|
|
gc_mutex: Mutex::new(false),
|
2018-12-17 12:00:39 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2018-12-27 09:11:11 +00:00
|
|
|
pub fn create_image_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<ImageIndexWriter, Error> {
|
2018-12-17 12:00:39 +00:00
|
|
|
|
2019-01-15 11:36:16 +00:00
|
|
|
let index = ImageIndexWriter::create(self.chunk_store.clone(), filename.as_ref(), size, chunk_size)?;
|
2018-12-17 12:00:39 +00:00
|
|
|
|
|
|
|
Ok(index)
|
|
|
|
}
|
|
|
|
|
2018-12-21 11:15:26 +00:00
|
|
|
pub fn open_image_reader<P: AsRef<Path>>(&self, filename: P) -> Result<ImageIndexReader, Error> {
|
2018-12-17 12:00:39 +00:00
|
|
|
|
2019-01-15 11:36:16 +00:00
|
|
|
let index = ImageIndexReader::open(self.chunk_store.clone(), filename.as_ref())?;
|
2018-12-17 12:00:39 +00:00
|
|
|
|
|
|
|
Ok(index)
|
|
|
|
}
|
2018-12-18 10:06:03 +00:00
|
|
|
|
2018-12-31 16:30:08 +00:00
|
|
|
pub fn create_archive_writer<P: AsRef<Path>>(
|
|
|
|
&self, filename: P,
|
|
|
|
chunk_size: usize
|
|
|
|
) -> Result<ArchiveIndexWriter, Error> {
|
|
|
|
|
2019-01-15 10:38:26 +00:00
|
|
|
let index = ArchiveIndexWriter::create(
|
|
|
|
self.chunk_store.clone(), filename.as_ref(), chunk_size)?;
|
2018-12-31 16:30:08 +00:00
|
|
|
|
|
|
|
Ok(index)
|
|
|
|
}
|
2019-01-18 11:01:37 +00:00
|
|
|
|
2019-01-02 13:27:04 +00:00
|
|
|
pub fn open_archive_reader<P: AsRef<Path>>(&self, filename: P) -> Result<ArchiveIndexReader, Error> {
|
|
|
|
|
2019-01-15 11:36:16 +00:00
|
|
|
let index = ArchiveIndexReader::open(self.chunk_store.clone(), filename.as_ref())?;
|
2019-01-02 13:27:04 +00:00
|
|
|
|
|
|
|
Ok(index)
|
|
|
|
}
|
|
|
|
|
2019-01-18 11:01:37 +00:00
|
|
|
pub fn base_path(&self) -> PathBuf {
|
|
|
|
self.chunk_store.base_path()
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn create_backup_dir(
|
|
|
|
&self,
|
|
|
|
backup_type: &str,
|
|
|
|
backup_id: &str,
|
|
|
|
backup_time: i64,
|
|
|
|
) -> Result<PathBuf, Error> {
|
|
|
|
let mut relative_path = PathBuf::new();
|
|
|
|
|
|
|
|
relative_path.push(backup_type);
|
|
|
|
|
|
|
|
relative_path.push(backup_id);
|
|
|
|
|
|
|
|
let dt = Utc.timestamp(backup_time, 0);
|
|
|
|
let date_str = dt.format("%Y-%m-%dT%H:%M:%S").to_string();
|
|
|
|
|
|
|
|
println!("date: {}", date_str);
|
|
|
|
|
|
|
|
relative_path.push(&date_str);
|
|
|
|
|
|
|
|
|
|
|
|
let mut full_path = self.base_path();
|
|
|
|
full_path.push(&relative_path);
|
|
|
|
|
|
|
|
std::fs::create_dir_all(&full_path)?;
|
|
|
|
|
|
|
|
Ok(relative_path)
|
|
|
|
}
|
|
|
|
|
2018-12-18 10:06:03 +00:00
|
|
|
pub fn list_images(&self) -> Result<Vec<PathBuf>, Error> {
|
2019-01-18 11:01:37 +00:00
|
|
|
let base = self.base_path();
|
2018-12-18 10:06:03 +00:00
|
|
|
|
|
|
|
let mut list = vec![];
|
|
|
|
|
2019-01-18 11:01:37 +00:00
|
|
|
// fixme: walk into subdirs ...
|
2018-12-18 10:06:03 +00:00
|
|
|
for entry in std::fs::read_dir(base)? {
|
|
|
|
let entry = entry?;
|
|
|
|
if entry.file_type()?.is_file() {
|
|
|
|
let path = entry.path();
|
|
|
|
if let Some(ext) = path.extension() {
|
2018-12-22 15:58:16 +00:00
|
|
|
if ext == "iidx" {
|
2018-12-18 10:06:03 +00:00
|
|
|
list.push(path);
|
2019-01-02 13:27:04 +00:00
|
|
|
} else if ext == "aidx" {
|
|
|
|
list.push(path);
|
2018-12-18 10:06:03 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(list)
|
|
|
|
}
|
|
|
|
|
2018-12-22 15:58:16 +00:00
|
|
|
fn mark_used_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> {
|
2018-12-18 10:06:03 +00:00
|
|
|
|
|
|
|
let image_list = self.list_images()?;
|
|
|
|
|
|
|
|
for path in image_list {
|
2019-01-02 13:27:04 +00:00
|
|
|
if let Some(ext) = path.extension() {
|
|
|
|
if ext == "iidx" {
|
2019-01-04 09:35:22 +00:00
|
|
|
let index = self.open_image_reader(&path)?;
|
2019-01-02 13:27:04 +00:00
|
|
|
index.mark_used_chunks(status)?;
|
|
|
|
} else if ext == "aidx" {
|
2019-01-04 09:35:22 +00:00
|
|
|
let index = self.open_archive_reader(&path)?;
|
2019-01-02 13:27:04 +00:00
|
|
|
index.mark_used_chunks(status)?;
|
|
|
|
}
|
|
|
|
}
|
2018-12-18 10:06:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2018-12-21 11:15:26 +00:00
|
|
|
pub fn garbage_collection(&self) -> Result<(), Error> {
|
2018-12-18 10:06:03 +00:00
|
|
|
|
2018-12-25 12:29:27 +00:00
|
|
|
if let Ok(ref mut _mutex) = self.gc_mutex.try_lock() {
|
2018-12-22 13:04:05 +00:00
|
|
|
|
2018-12-22 15:58:16 +00:00
|
|
|
let mut gc_status = GarbageCollectionStatus::default();
|
|
|
|
gc_status.used_bytes = 0;
|
2018-12-18 10:18:55 +00:00
|
|
|
|
2018-12-22 15:58:16 +00:00
|
|
|
println!("Start GC phase1 (mark chunks)");
|
|
|
|
|
|
|
|
self.mark_used_chunks(&mut gc_status)?;
|
|
|
|
|
|
|
|
println!("Start GC phase2 (sweep unused chunks)");
|
2019-01-02 13:27:04 +00:00
|
|
|
self.chunk_store.sweep_unused_chunks(&mut gc_status)?;
|
2018-12-22 15:58:16 +00:00
|
|
|
|
|
|
|
println!("Used bytes: {}", gc_status.used_bytes);
|
|
|
|
println!("Used chunks: {}", gc_status.used_chunks);
|
|
|
|
println!("Disk bytes: {}", gc_status.disk_bytes);
|
|
|
|
println!("Disk chunks: {}", gc_status.disk_chunks);
|
|
|
|
|
|
|
|
} else {
|
|
|
|
println!("Start GC failed - (already running/locked)");
|
|
|
|
}
|
2018-12-18 10:06:03 +00:00
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
2018-12-17 12:00:39 +00:00
|
|
|
}
|