tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

tools: remove io module

Browse Source 
now completely replaced by proxmox::tools::io.

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller 2019-07-01 10:44:11 +02:00
parent 9b2b627fe0
commit d0162d53d3
3 changed files with 0 additions and 538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/tools.rs
View File

@ -37,7 +37,6 @@ pub mod procfs;
pub mod acl;
pub mod xattr;
pub mod vec;
pub mod io;
pub mod futures;
mod process_locker;

311
src/tools/io.rs
View File

@ -1,311 +0,0 @@
//! Module providing I/O helpers (sync and async).
//!
//! The [`ops`](io::ops) module provides helper traits for types implementing [`Read`](std::io::Read).
//!
//! The top level functions in of this module here are used for standalone implementations of
//! various functionality which is actually intended to be available as methods to types
//! implementing `AsyncRead`, which, however, without async/await cannot be methods due to them
//! having non-static lifetimes in that case.
//!
//! ```ignore
//! use std::io;
//!
//! use crate::tools::io::read_exact_allocated;
//! use crate::tools::vec::{self, ops::*};
//!
//! // Currently usable:
//! fn do_something() -> impl Future<Item = Vec<u8>, Error = io::Error> {
//! tokio::fs::File::open("some.file")
//! .and_then(|file| read_exact_allocated(file, unsafe { vec::uninitialized(1024) }))
//! .and_then(|(file, mut buffer)| {
//! so_something_with(&buffer);
//! // append more data:
//! tokio::io::read_exact(file, unsafe { buffer.grow_uninitialized(1024) })
//! })
//! .and_then(|(_file, bigger_buffer)| {
//! // use bigger_buffer
//! Ok(())
//! });
//! }
//!
//! // Future async/await variant:
//! async fn do_something() -> Vec<u8> {
//! let mut file = tokio::fs::File::open("some.file").await?;
//! let mut buffer = file.read_exact_allocated(1024).await?;
//! do_something_with(buffer);
//! file.append_to_vec(&mut buffer, 1024).await?;
//! buffer
//! }
//! ```
use std::io;
use futures::Future;
use futures::{Async, Poll};
use tokio::io::AsyncRead;
use crate::tools::vec::{self, ops::*};
pub mod ops;
/// Create a future which reads an exact amount of bytes from an input.
///
/// The future's output is a tuple containing the input and a newly allocated `Vec<u8>` containing
/// the data.
///
/// Example:
/// ```
/// # use futures::future::Future;
/// # use proxmox_backup::tools::io::*;
/// tokio::fs::File::open("some.file")
/// .and_then(|file| read_exact_allocated(file, 1024))
/// .and_then(|(_file, data)| {
/// // use data
/// Ok(())
/// });
/// ```
pub fn read_exact_allocated<R: AsyncRead>(reader: R, size: usize) -> ReadExactAllocated<R> {
ReadExactAllocated(Some(reader), None, size)
}
/// A future returned by [`read_exact_allocated`].
pub struct ReadExactAllocated<R: AsyncRead>(Option<R>, Option<Vec<u8>>, usize);
impl<R: AsyncRead> Future for ReadExactAllocated<R> {
type Item = (R, Vec<u8>);
type Error = io::Error;
fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
assert!(self.0.is_some(), "polled after ready");
// allocation happens on first poll:
if self.1.is_none() {
self.1 = Some(unsafe { vec::uninitialized(self.2) });
// now self.2 is the position:
self.2 = 0;
}
let mut buffer = self.1.take().unwrap();
loop {
match self.0.as_mut().unwrap().poll_read(&mut buffer[self.2..]) {
Ok(Async::Ready(0)) => {
self.0 = None;
return Err(io::Error::from(io::ErrorKind::UnexpectedEof));
}
Ok(Async::Ready(some)) => {
self.2 += some;
if self.2 == buffer.len() {
self.0 = None;
return Ok(Async::Ready((self.0.take().unwrap(), buffer)));
}
continue;
}
Ok(Async::NotReady) => {
self.1 = Some(buffer);
return Ok(Async::NotReady);
}
Err(err) => {
self.0 = None;
return Err(err);
}
}
}
}
}
/// Create a future which appends up to at most `size` bytes to a vector, growing it as needed.
///
/// This will grow the vector as if a single `.reserve(amount_to_read)` call was made and fill it
/// with as much data as a single read call will provide.
///
/// The future's output is a tuple containing the input, the vector and the number of bytes
/// actually read.
///
/// Example:
/// ```
/// # use futures::future::Future;
/// # use proxmox_backup::tools::io::*;
/// tokio::fs::File::open("some.file")
/// .and_then(|file| append_to_vec(file, Vec::new(), 1024))
/// .and_then(|(_file, data, size)| {
/// assert!(data.len() == size);
/// println!("Actually got {} bytes of data.", size);
/// // use the data
/// Ok(())
/// });
/// ```
pub fn append_to_vec<R, V>(reader: R, mut vector: V, size: usize) -> AppendToVec<R, V>
where
R: AsyncRead,
V: AsMut<Vec<u8>>,
{
let pos = vector.as_mut().len();
unsafe {
vector.as_mut().grow_uninitialized(size);
}
AppendToVec(Some(reader), Some(vector), pos)
}
pub struct AppendToVec<R, V>(Option<R>, Option<V>, usize)
where
R: AsyncRead,
V: AsMut<Vec<u8>>;
impl<R, V> Future for AppendToVec<R, V>
where
R: AsyncRead,
V: AsMut<Vec<u8>>,
{
type Item = (R, V, usize);
type Error = io::Error;
fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
assert!(self.0.is_some() && self.1.is_some(), "polled after ready");
let mut output = self.1.take().unwrap();
match self.0.as_mut().unwrap().poll_read(&mut output.as_mut()[self.2..]) {
Ok(Async::Ready(some)) => {
unsafe {
output.as_mut().set_len(self.2 + some);
}
return Ok(Async::Ready((self.0.take().unwrap(), output, some)));
}
Ok(Async::NotReady) => {
self.1 = Some(output);
return Ok(Async::NotReady);
}
Err(err) => {
self.0 = None;
return Err(err);
}
}
}
}
/// Create a future which appends an exact amount of bytes to a vector, growing it as needed.
///
/// This will grow the vector as if a single `.reserve(amount_to_read)` call was made and fill it
/// as much data as requested. If not enough data is available, this produces an
/// [`io::Error`](std::io::Error) of kind
/// [`ErrorKind::UnexpectedEof`](std::io::ErrorKind::UnexpectedEof).
///
/// The future's output is a tuple containing the input and the vector.
///
/// Example:
/// ```
/// # use futures::future::Future;
/// # use proxmox_backup::tools::io::*;
/// tokio::fs::File::open("some.file")
/// .and_then(|file| append_exact_to_vec(file, Vec::new(), 1024))
/// .and_then(|(_file, data)| {
/// assert!(data.len() == 1024);
/// // use data
/// Ok(())
/// });
/// ```
pub fn append_exact_to_vec<R, V>(reader: R, mut vector: V, size: usize) -> AppendExactToVec<R, V>
where
R: AsyncRead,
V: AsMut<Vec<u8>>,
{
let pos = vector.as_mut().len();
unsafe {
vector.as_mut().grow_uninitialized(size);
}
AppendExactToVec(Some(reader), Some(vector), pos)
}
pub struct AppendExactToVec<R, V>(Option<R>, Option<V>, usize)
where
R: AsyncRead,
V: AsMut<Vec<u8>>;
impl<R, V> Future for AppendExactToVec<R, V>
where
R: AsyncRead,
V: AsMut<Vec<u8>>,
{
type Item = (R, V);
type Error = io::Error;
fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
assert!(self.0.is_some() && self.1.is_some(), "polled after ready");
let mut output = self.1.take().unwrap();
loop {
match self.0.as_mut().unwrap().poll_read(&mut output.as_mut()[self.2..]) {
Ok(Async::Ready(0)) => {
self.0 = None;
return Err(io::Error::from(io::ErrorKind::UnexpectedEof));
}
Ok(Async::Ready(some)) => {
self.2 += some;
if self.2 == output.as_mut().len() {
self.0 = None;
return Ok(Async::Ready((self.0.take().unwrap(), output)));
}
continue;
}
Ok(Async::NotReady) => {
self.1 = Some(output);
return Ok(Async::NotReady);
}
Err(err) => {
self.0 = None;
return Err(err);
}
}
}
}
}
/*
* TODO: A trait such as the one below is only useful inside `async fn`, so this partwill have to
* wait...
*
* When we have async/await we can finish this and move it into io/async_read.rs
/// Some additional related functionality for types implementing `AsyncRead`. Note that most of
/// these methods map to functions from the [`io`](super::io) module, which are standalone
/// variants.
///
/// This trait only works with standard futures or as part of `poll_fn` bodies, due to it requiring
/// non-static lifetimes on futures.
pub trait AsyncReadExtOps: AsyncRead + Sized {
/// Read data into a newly allocated vector. This is a shortcut for:
/// ```
/// let mut data = Vec::with_capacity(len);
/// unsafe {
/// data.set_len(len);
/// }
/// reader.read_exact(&mut data)
/// ```
///
/// With this trait, we just use:
/// ```
/// use crate::tools::vec::ops::*;
///
/// let data = reader.read_exact_allocated(len).await?;
/// ```
fn read_exact_allocated(&mut self, size: usize) -> ReadExactAllocated<&mut Self> {
ReadExactAllocated(crate::tools::io::read_exact_allocated(self, size))
}
}
impl<T: AsyncRead + Sized> AsyncReadExtOps for T {
}
pub struct ReadExactAllocated<R: AsyncRead>(crate::tools::io::ReadExactAllocated<R>);
impl<R: AsyncRead> futures::Future for ReadExactAllocated<R> {
type Item = Vec<u8>;
type Error = io::Error;
fn poll(&mut self) -> futures::Poll<Self::Item, Self::Error> {
let (_this, data) = futures::try_ready!(self.0.poll());
Ok(futures::Async::Ready(data))
}
}
*/

226
src/tools/io/ops.rs
View File

@ -1,226 +0,0 @@
//! This module provides additional operations for handling byte buffers for types implementing
//! [`Read`](std::io::Read).
//!
//! See the [`ReadExtOps`](ops::ReadExtOps) trait for examples.
use std::io;
use endian_trait::Endian;
use crate::tools::vec::{self, ops::*};
/// Adds some additional related functionality for types implementing [`Read`](std::io::Read).
///
/// Particularly for reading into a newly allocated buffer, appending to a `Vec<u8>` or reading
/// values of a specific endianess (types implementing [`Endian`]).
///
/// Examples:
/// ```no_run
/// use proxmox_backup::tools::io::ops::*;
///
/// # fn code() -> std::io::Result<()> {
/// let mut file = std::fs::File::open("some.data")?;
///
/// // read some bytes into a newly allocated Vec<u8>:
/// let mut data = file.read_exact_allocated(64)?;
///
/// // appending data to a vector:
/// let actually_appended = file.append_to_vec(&mut data, 64)?; // .read() version
/// file.append_exact_to_vec(&mut data, 64)?; // .read_exact() version
/// # Ok(())
/// # }
/// ```
///
/// Or for reading values of a defined representation and endianess:
///
/// ```no_run
/// # use endian_trait::Endian;
/// # use proxmox_backup::tools::io::ops::*;
///
/// #[derive(Endian)]
/// #[repr(C)]
/// struct Header {
/// version: u16,
/// data_size: u16,
/// }
///
/// # fn code(mut file: std::fs::File) -> std::io::Result<()> {
/// // We have given `Header` a proper binary representation via `#[repr]`, so this is safe:
/// let header: Header = unsafe { file.read_le_value()? };
/// let mut blob = file.read_exact_allocated(header.data_size as usize)?;
/// # Ok(())
/// # }
/// ```
///
/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
pub trait ReadExtOps {
/// Read data into a newly allocated vector. This is a shortcut for:
/// ```ignore
/// let mut data = Vec::with_capacity(len);
/// unsafe {
/// data.set_len(len);
/// }
/// reader.read_exact(&mut data)?;
/// ```
///
/// With this trait, we just use:
/// ```no_run
/// use proxmox_backup::tools::io::ops::*;
/// # fn code(mut reader: std::fs::File, len: usize) -> std::io::Result<()> {
/// let data = reader.read_exact_allocated(len)?;
/// # Ok(())
/// # }
/// ```
fn read_exact_allocated(&mut self, size: usize) -> io::Result<Vec<u8>>;
/// Append data to a vector, growing it as necessary. Returns the amount of data appended.
fn append_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<usize>;
/// Append an exact amount of data to a vector, growing it as necessary.
fn append_exact_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<()>;
/// Read a value with host endianess.
///
/// This is limited to types implementing the [`Endian`] trait under the assumption that
/// this is only done for types which are supposed to be read/writable directly.
///
/// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
/// this is considered unsafe.
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox_backup::tools::io::ops::*;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
/// struct Data {
/// value: u16,
/// count: u32,
/// }
///
/// # fn code() -> std::io::Result<()> {
/// let mut file = std::fs::File::open("my-raw.dat")?;
/// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
/// // safely use our helper:
/// let data: Data = unsafe { file.read_host_value()? };
/// # Ok(())
/// # }
/// ```
///
/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
unsafe fn read_host_value<T: Endian>(&mut self) -> io::Result<T>;
/// Read a little endian value.
///
/// The return type is required to implement the [`Endian`] trait, and we make the
/// assumption that this is only done for types which are supposed to be read/writable
/// directly.
///
/// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
/// this is considered unsafe.
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox_backup::tools::io::ops::*;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
/// struct Data {
/// value: u16,
/// count: u32,
/// }
///
/// # fn code() -> std::io::Result<()> {
/// let mut file = std::fs::File::open("my-little-endian.dat")?;
/// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
/// // safely use our helper:
/// let data: Data = unsafe { file.read_le_value()? };
/// # Ok(())
/// # }
/// ```
///
/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
unsafe fn read_le_value<T: Endian>(&mut self) -> io::Result<T>;
/// Read a big endian value.
///
/// The return type is required to implement the [`Endian`] trait, and we make the
/// assumption that this is only done for types which are supposed to be read/writable
/// directly.
///
/// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore
/// this is considered unsafe.
///
/// ```no_run
/// # use endian_trait::Endian;
/// use proxmox_backup::tools::io::ops::*;
///
/// #[derive(Endian)]
/// #[repr(C, packed)]
/// struct Data {
/// value: u16,
/// count: u32,
/// }
///
/// # fn code() -> std::io::Result<()> {
/// let mut file = std::fs::File::open("my-big-endian.dat")?;
/// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can
/// // safely use our helper:
/// let data: Data = unsafe { file.read_be_value()? };
/// # Ok(())
/// # }
/// ```
///
/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html
unsafe fn read_be_value<T: Endian>(&mut self) -> io::Result<T>;
}
impl<R: io::Read> ReadExtOps for R {
fn read_exact_allocated(&mut self, size: usize) -> io::Result<Vec<u8>> {
let mut out = unsafe { vec::uninitialized(size) };
self.read_exact(&mut out)?;
Ok(out)
}
fn append_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<usize> {
let pos = out.len();
unsafe {
out.grow_uninitialized(size);
}
let got = self.read(&mut out[pos..])?;
unsafe {
out.set_len(pos + got);
}
Ok(got)
}
fn append_exact_to_vec(&mut self, out: &mut Vec<u8>, size: usize) -> io::Result<()> {
let pos = out.len();
unsafe {
out.grow_uninitialized(size);
}
self.read_exact(&mut out[pos..])?;
Ok(())
}
unsafe fn read_host_value<T: Endian>(&mut self) -> io::Result<T> {
let mut value: T = std::mem::uninitialized();
self.read_exact(std::slice::from_raw_parts_mut(
&mut value as *mut T as *mut u8,
std::mem::size_of::<T>(),
))?;
Ok(value)
}
unsafe fn read_le_value<T: Endian>(&mut self) -> io::Result<T> {
Ok(self.read_host_value::<T>()?.
from_le()
)
}
unsafe fn read_be_value<T: Endian>(&mut self) -> io::Result<T> {
Ok(self.read_host_value::<T>()?
.from_be()
)
}
}