server/rest: rust format

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
This commit is contained in:
Thomas Lamprecht 2021-03-29 08:17:26 +02:00
parent 6bc87d3952
commit d43c407a00

View File

@ -9,48 +9,42 @@ use std::task::{Context, Poll};
use anyhow::{bail, format_err, Error}; use anyhow::{bail, format_err, Error};
use futures::future::{self, FutureExt, TryFutureExt}; use futures::future::{self, FutureExt, TryFutureExt};
use futures::stream::TryStreamExt; use futures::stream::TryStreamExt;
use hyper::header::{self, HeaderMap};
use hyper::body::HttpBody; use hyper::body::HttpBody;
use hyper::header::{self, HeaderMap};
use hyper::http::request::Parts; use hyper::http::request::Parts;
use hyper::{Body, Request, Response, StatusCode}; use hyper::{Body, Request, Response, StatusCode};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use percent_encoding::percent_decode_str;
use regex::Regex;
use serde_json::{json, Value}; use serde_json::{json, Value};
use tokio::fs::File; use tokio::fs::File;
use tokio::time::Instant; use tokio::time::Instant;
use percent_encoding::percent_decode_str;
use url::form_urlencoded; use url::form_urlencoded;
use regex::Regex;
use proxmox::http_err;
use proxmox::api::{
ApiHandler,
ApiMethod,
HttpError,
Permission,
RpcEnvironment,
RpcEnvironmentType,
check_api_permission,
};
use proxmox::api::schema::{ use proxmox::api::schema::{
ObjectSchemaType, parse_parameter_strings, parse_simple_value, verify_json_object, ObjectSchemaType,
ParameterSchema, ParameterSchema,
parse_parameter_strings,
parse_simple_value,
verify_json_object,
}; };
use proxmox::api::{
check_api_permission, ApiHandler, ApiMethod, HttpError, Permission, RpcEnvironment,
RpcEnvironmentType,
};
use proxmox::http_err;
use super::environment::RestEnvironment; use super::environment::RestEnvironment;
use super::formatter::*; use super::formatter::*;
use super::ApiConfig; use super::ApiConfig;
use crate::auth_helpers::*;
use crate::api2::types::{Authid, Userid}; use crate::api2::types::{Authid, Userid};
use crate::tools; use crate::auth_helpers::*;
use crate::tools::FileLogger;
use crate::tools::ticket::Ticket;
use crate::config::cached_user_info::CachedUserInfo; use crate::config::cached_user_info::CachedUserInfo;
use crate::tools;
use crate::tools::ticket::Ticket;
use crate::tools::FileLogger;
extern "C" { fn tzset(); } extern "C" {
fn tzset();
}
pub struct RestServer { pub struct RestServer {
pub api_config: Arc<ApiConfig>, pub api_config: Arc<ApiConfig>,
@ -59,13 +53,16 @@ pub struct RestServer {
const MAX_URI_QUERY_LENGTH: usize = 3072; const MAX_URI_QUERY_LENGTH: usize = 3072;
impl RestServer { impl RestServer {
pub fn new(api_config: ApiConfig) -> Self { pub fn new(api_config: ApiConfig) -> Self {
Self { api_config: Arc::new(api_config) } Self {
api_config: Arc::new(api_config),
}
} }
} }
impl tower_service::Service<&Pin<Box<tokio_openssl::SslStream<tokio::net::TcpStream>>>> for RestServer { impl tower_service::Service<&Pin<Box<tokio_openssl::SslStream<tokio::net::TcpStream>>>>
for RestServer
{
type Response = ApiService; type Response = ApiService;
type Error = Error; type Error = Error;
type Future = Pin<Box<dyn Future<Output = Result<ApiService, Error>> + Send>>; type Future = Pin<Box<dyn Future<Output = Result<ApiService, Error>> + Send>>;
@ -74,14 +71,17 @@ impl tower_service::Service<&Pin<Box<tokio_openssl::SslStream<tokio::net::TcpStr
Poll::Ready(Ok(())) Poll::Ready(Ok(()))
} }
fn call(&mut self, ctx: &Pin<Box<tokio_openssl::SslStream<tokio::net::TcpStream>>>) -> Self::Future { fn call(
&mut self,
ctx: &Pin<Box<tokio_openssl::SslStream<tokio::net::TcpStream>>>,
) -> Self::Future {
match ctx.get_ref().peer_addr() { match ctx.get_ref().peer_addr() {
Err(err) => { Err(err) => future::err(format_err!("unable to get peer address - {}", err)).boxed(),
future::err(format_err!("unable to get peer address - {}", err)).boxed() Ok(peer) => future::ok(ApiService {
} peer,
Ok(peer) => { api_config: self.api_config.clone(),
future::ok(ApiService { peer, api_config: self.api_config.clone() }).boxed() })
} .boxed(),
} }
} }
} }
@ -97,12 +97,12 @@ impl tower_service::Service<&tokio::net::TcpStream> for RestServer {
fn call(&mut self, ctx: &tokio::net::TcpStream) -> Self::Future { fn call(&mut self, ctx: &tokio::net::TcpStream) -> Self::Future {
match ctx.peer_addr() { match ctx.peer_addr() {
Err(err) => { Err(err) => future::err(format_err!("unable to get peer address - {}", err)).boxed(),
future::err(format_err!("unable to get peer address - {}", err)).boxed() Ok(peer) => future::ok(ApiService {
} peer,
Ok(peer) => { api_config: self.api_config.clone(),
future::ok(ApiService { peer, api_config: self.api_config.clone() }).boxed() })
} .boxed(),
} }
} }
} }
@ -122,8 +122,9 @@ impl tower_service::Service<&tokio::net::UnixStream> for RestServer {
let fake_peer = "0.0.0.0:807".parse().unwrap(); let fake_peer = "0.0.0.0:807".parse().unwrap();
future::ok(ApiService { future::ok(ApiService {
peer: fake_peer, peer: fake_peer,
api_config: self.api_config.clone() api_config: self.api_config.clone(),
}).boxed() })
.boxed()
} }
} }
@ -140,8 +141,9 @@ fn log_response(
resp: &Response<Body>, resp: &Response<Body>,
user_agent: Option<String>, user_agent: Option<String>,
) { ) {
if resp.extensions().get::<NoLogExtension>().is_some() {
if resp.extensions().get::<NoLogExtension>().is_some() { return; }; return;
};
// we also log URL-to-long requests, so avoid message bigger than PIPE_BUF (4k on Linux) // we also log URL-to-long requests, so avoid message bigger than PIPE_BUF (4k on Linux)
// to profit from atomicty guarantees for O_APPEND opened logfiles // to profit from atomicty guarantees for O_APPEND opened logfiles
@ -157,7 +159,15 @@ fn log_response(
message = &data.0; message = &data.0;
} }
log::error!("{} {}: {} {}: [client {}] {}", method.as_str(), path, status.as_str(), reason, peer, message); log::error!(
"{} {}: {} {}: [client {}] {}",
method.as_str(),
path,
status.as_str(),
reason,
peer,
message
);
} }
if let Some(logfile) = logfile { if let Some(logfile) = logfile {
let auth_id = match resp.extensions().get::<Authid>() { let auth_id = match resp.extensions().get::<Authid>() {
@ -169,10 +179,7 @@ fn log_response(
let datetime = proxmox::tools::time::strftime_local("%d/%m/%Y:%H:%M:%S %z", now) let datetime = proxmox::tools::time::strftime_local("%d/%m/%Y:%H:%M:%S %z", now)
.unwrap_or_else(|_| "-".to_string()); .unwrap_or_else(|_| "-".to_string());
logfile logfile.lock().unwrap().log(format!(
.lock()
.unwrap()
.log(format!(
"{} - {} [{}] \"{} {}\" {} {} {}", "{} - {} [{}] \"{} {}\" {} {} {}",
peer.ip(), peer.ip(),
auth_id, auth_id,
@ -208,11 +215,13 @@ fn get_proxied_peer(headers: &HeaderMap) -> Option<std::net::SocketAddr> {
fn get_user_agent(headers: &HeaderMap) -> Option<String> { fn get_user_agent(headers: &HeaderMap) -> Option<String> {
let agent = headers.get(header::USER_AGENT)?.to_str(); let agent = headers.get(header::USER_AGENT)?.to_str();
agent.map(|s| { agent
.map(|s| {
let mut s = s.to_owned(); let mut s = s.to_owned();
s.truncate(128); s.truncate(128);
s s
}).ok() })
.ok()
} }
impl tower_service::Service<Request<Body>> for ApiService { impl tower_service::Service<Request<Body>> for ApiService {
@ -260,7 +269,6 @@ fn parse_query_parameters<S: 'static + BuildHasher + Send>(
parts: &Parts, parts: &Parts,
uri_param: &HashMap<String, String, S>, uri_param: &HashMap<String, String, S>,
) -> Result<Value, Error> { ) -> Result<Value, Error> {
let mut param_list: Vec<(String, String)> = vec![]; let mut param_list: Vec<(String, String)> = vec![];
if !form.is_empty() { if !form.is_empty() {
@ -271,7 +279,9 @@ fn parse_query_parameters<S: 'static + BuildHasher + Send>(
if let Some(query_str) = parts.uri.query() { if let Some(query_str) = parts.uri.query() {
for (k, v) in form_urlencoded::parse(query_str.as_bytes()).into_owned() { for (k, v) in form_urlencoded::parse(query_str.as_bytes()).into_owned() {
if k == "_dc" { continue; } // skip extjs "disable cache" parameter if k == "_dc" {
continue;
} // skip extjs "disable cache" parameter
param_list.push((k, v)); param_list.push((k, v));
} }
} }
@ -291,7 +301,6 @@ async fn get_request_parameters<S: 'static + BuildHasher + Send>(
req_body: Body, req_body: Body,
uri_param: HashMap<String, String, S>, uri_param: HashMap<String, String, S>,
) -> Result<Value, Error> { ) -> Result<Value, Error> {
let mut is_json = false; let mut is_json = false;
if let Some(value) = parts.headers.get(header::CONTENT_TYPE) { if let Some(value) = parts.headers.get(header::CONTENT_TYPE) {
@ -309,16 +318,18 @@ async fn get_request_parameters<S: 'static + BuildHasher + Send>(
let body = req_body let body = req_body
.map_err(|err| http_err!(BAD_REQUEST, "Promlems reading request body: {}", err)) .map_err(|err| http_err!(BAD_REQUEST, "Promlems reading request body: {}", err))
.try_fold(Vec::new(), |mut acc, chunk| async move { .try_fold(Vec::new(), |mut acc, chunk| async move {
if acc.len() + chunk.len() < 64*1024 { //fimxe: max request body size? // FIXME: max request body size?
if acc.len() + chunk.len() < 64 * 1024 {
acc.extend_from_slice(&*chunk); acc.extend_from_slice(&*chunk);
Ok(acc) Ok(acc)
} else { } else {
Err(http_err!(BAD_REQUEST, "Request body too large")) Err(http_err!(BAD_REQUEST, "Request body too large"))
} }
}).await?; })
.await?;
let utf8_data = std::str::from_utf8(&body) let utf8_data =
.map_err(|err| format_err!("Request body not uft8: {}", err))?; std::str::from_utf8(&body).map_err(|err| format_err!("Request body not uft8: {}", err))?;
if is_json { if is_json {
let mut params: Value = serde_json::from_str(utf8_data)?; let mut params: Value = serde_json::from_str(utf8_data)?;
@ -342,7 +353,6 @@ async fn proxy_protected_request(
req_body: Body, req_body: Body,
peer: &std::net::SocketAddr, peer: &std::net::SocketAddr,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
let mut uri_parts = parts.uri.clone().into_parts(); let mut uri_parts = parts.uri.clone().into_parts();
uri_parts.scheme = Some(http::uri::Scheme::HTTP); uri_parts.scheme = Some(http::uri::Scheme::HTTP);
@ -352,9 +362,10 @@ async fn proxy_protected_request(
parts.uri = new_uri; parts.uri = new_uri;
let mut request = Request::from_parts(parts, req_body); let mut request = Request::from_parts(parts, req_body);
request request.headers_mut().insert(
.headers_mut() header::FORWARDED,
.insert(header::FORWARDED, format!("for=\"{}\";", peer).parse().unwrap()); format!("for=\"{}\";", peer).parse().unwrap(),
);
let reload_timezone = info.reload_timezone; let reload_timezone = info.reload_timezone;
@ -367,7 +378,11 @@ async fn proxy_protected_request(
}) })
.await?; .await?;
if reload_timezone { unsafe { tzset(); } } if reload_timezone {
unsafe {
tzset();
}
}
Ok(resp) Ok(resp)
} }
@ -380,7 +395,6 @@ pub async fn handle_api_request<Env: RpcEnvironment, S: 'static + BuildHasher +
req_body: Body, req_body: Body,
uri_param: HashMap<String, String, S>, uri_param: HashMap<String, String, S>,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
let delay_unauth_time = std::time::Instant::now() + std::time::Duration::from_millis(3000); let delay_unauth_time = std::time::Instant::now() + std::time::Duration::from_millis(3000);
let result = match info.handler { let result = match info.handler {
@ -389,12 +403,13 @@ pub async fn handle_api_request<Env: RpcEnvironment, S: 'static + BuildHasher +
(handler)(parts, req_body, params, info, Box::new(rpcenv)).await (handler)(parts, req_body, params, info, Box::new(rpcenv)).await
} }
ApiHandler::Sync(handler) => { ApiHandler::Sync(handler) => {
let params = get_request_parameters(info.parameters, parts, req_body, uri_param).await?; let params =
(handler)(params, info, &mut rpcenv) get_request_parameters(info.parameters, parts, req_body, uri_param).await?;
.map(|data| (formatter.format_data)(data, &rpcenv)) (handler)(params, info, &mut rpcenv).map(|data| (formatter.format_data)(data, &rpcenv))
} }
ApiHandler::Async(handler) => { ApiHandler::Async(handler) => {
let params = get_request_parameters(info.parameters, parts, req_body, uri_param).await?; let params =
get_request_parameters(info.parameters, parts, req_body, uri_param).await?;
(handler)(params, info, &mut rpcenv) (handler)(params, info, &mut rpcenv)
.await .await
.map(|data| (formatter.format_data)(data, &rpcenv)) .map(|data| (formatter.format_data)(data, &rpcenv))
@ -413,7 +428,11 @@ pub async fn handle_api_request<Env: RpcEnvironment, S: 'static + BuildHasher +
} }
}; };
if info.reload_timezone { unsafe { tzset(); } } if info.reload_timezone {
unsafe {
tzset();
}
}
Ok(resp) Ok(resp)
} }
@ -425,7 +444,6 @@ fn get_index(
api: &Arc<ApiConfig>, api: &Arc<ApiConfig>,
parts: Parts, parts: Parts,
) -> Response<Body> { ) -> Response<Body> {
let nodename = proxmox::tools::nodename(); let nodename = proxmox::tools::nodename();
let user = userid.as_ref().map(|u| u.as_str()).unwrap_or(""); let user = userid.as_ref().map(|u| u.as_str()).unwrap_or("");
@ -462,9 +480,7 @@ fn get_index(
let (ct, index) = match api.render_template(template_file, &data) { let (ct, index) = match api.render_template(template_file, &data) {
Ok(index) => ("text/html", index), Ok(index) => ("text/html", index),
Err(err) => { Err(err) => ("text/plain", format!("Error rendering template: {}", err)),
("text/plain", format!("Error rendering template: {}", err))
}
}; };
let mut resp = Response::builder() let mut resp = Response::builder()
@ -481,7 +497,6 @@ fn get_index(
} }
fn extension_to_content_type(filename: &Path) -> (&'static str, bool) { fn extension_to_content_type(filename: &Path) -> (&'static str, bool) {
if let Some(ext) = filename.extension().and_then(|osstr| osstr.to_str()) { if let Some(ext) = filename.extension().and_then(|osstr| osstr.to_str()) {
return match ext { return match ext {
"css" => ("text/css", false), "css" => ("text/css", false),
@ -510,7 +525,6 @@ fn extension_to_content_type(filename: &Path) -> (&'static str, bool) {
} }
async fn simple_static_file_download(filename: PathBuf) -> Result<Response<Body>, Error> { async fn simple_static_file_download(filename: PathBuf) -> Result<Response<Body>, Error> {
let (content_type, _nocomp) = extension_to_content_type(&filename); let (content_type, _nocomp) = extension_to_content_type(&filename);
use tokio::io::AsyncReadExt; use tokio::io::AsyncReadExt;
@ -527,7 +541,8 @@ async fn simple_static_file_download(filename: PathBuf) -> Result<Response<Body>
let mut response = Response::new(data.into()); let mut response = Response::new(data.into());
response.headers_mut().insert( response.headers_mut().insert(
header::CONTENT_TYPE, header::CONTENT_TYPE,
header::HeaderValue::from_static(content_type)); header::HeaderValue::from_static(content_type),
);
Ok(response) Ok(response)
} }
@ -542,17 +557,15 @@ async fn chuncked_static_file_download(filename: PathBuf) -> Result<Response<Bod
.map_ok(|bytes| bytes.freeze()); .map_ok(|bytes| bytes.freeze());
let body = Body::wrap_stream(payload); let body = Body::wrap_stream(payload);
// fixme: set other headers ? // FIXME: set other headers ?
Ok(Response::builder() Ok(Response::builder()
.status(StatusCode::OK) .status(StatusCode::OK)
.header(header::CONTENT_TYPE, content_type) .header(header::CONTENT_TYPE, content_type)
.body(body) .body(body)
.unwrap() .unwrap())
)
} }
async fn handle_static_file_download(filename: PathBuf) -> Result<Response<Body>, Error> { async fn handle_static_file_download(filename: PathBuf) -> Result<Response<Body>, Error> {
let metadata = tokio::fs::metadata(filename.clone()) let metadata = tokio::fs::metadata(filename.clone())
.map_err(|err| http_err!(BAD_REQUEST, "File access problems: {}", err)) .map_err(|err| http_err!(BAD_REQUEST, "File access problems: {}", err))
.await?; .await?;
@ -592,10 +605,7 @@ fn extract_auth_data(headers: &http::HeaderMap) -> Option<AuthData> {
Some(Ok(v)) => Some(v.to_owned()), Some(Ok(v)) => Some(v.to_owned()),
_ => None, _ => None,
}; };
return Some(AuthData::User(UserAuthData { return Some(AuthData::User(UserAuthData { ticket, csrf_token }));
ticket,
csrf_token,
}));
} }
} }
} }
@ -607,7 +617,7 @@ fn extract_auth_data(headers: &http::HeaderMap) -> Option<AuthData> {
} else { } else {
None None
} }
}, }
_ => None, _ => None,
} }
} }
@ -633,17 +643,24 @@ fn check_auth(
if method != hyper::Method::GET { if method != hyper::Method::GET {
if let Some(csrf_token) = &user_auth_data.csrf_token { if let Some(csrf_token) = &user_auth_data.csrf_token {
verify_csrf_prevention_token(csrf_secret(), &userid, &csrf_token, -300, ticket_lifetime)?; verify_csrf_prevention_token(
csrf_secret(),
&userid,
&csrf_token,
-300,
ticket_lifetime,
)?;
} else { } else {
bail!("missing CSRF prevention token"); bail!("missing CSRF prevention token");
} }
} }
Ok(auth_id) Ok(auth_id)
}, }
AuthData::ApiToken(api_token) => { AuthData::ApiToken(api_token) => {
let mut parts = api_token.splitn(2, ':'); let mut parts = api_token.splitn(2, ':');
let tokenid = parts.next() let tokenid = parts
.next()
.ok_or_else(|| format_err!("failed to split API token header"))?; .ok_or_else(|| format_err!("failed to split API token header"))?;
let tokenid: Authid = tokenid.parse()?; let tokenid: Authid = tokenid.parse()?;
@ -651,7 +668,8 @@ fn check_auth(
bail!("user account or token disabled or expired."); bail!("user account or token disabled or expired.");
} }
let tokensecret = parts.next() let tokensecret = parts
.next()
.ok_or_else(|| format_err!("failed to split API token header"))?; .ok_or_else(|| format_err!("failed to split API token header"))?;
let tokensecret = percent_decode_str(tokensecret) let tokensecret = percent_decode_str(tokensecret)
.decode_utf8() .decode_utf8()
@ -669,7 +687,6 @@ async fn handle_request(
req: Request<Body>, req: Request<Body>,
peer: &std::net::SocketAddr, peer: &std::net::SocketAddr,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
let (parts, body) = req.into_parts(); let (parts, body) = req.into_parts();
let method = parts.method.clone(); let method = parts.method.clone();
let (path, components) = tools::normalize_uri_path(parts.uri.path())?; let (path, components) = tools::normalize_uri_path(parts.uri.path())?;
@ -695,9 +712,7 @@ async fn handle_request(
let access_forbidden_time = std::time::Instant::now() + std::time::Duration::from_millis(500); let access_forbidden_time = std::time::Instant::now() + std::time::Duration::from_millis(500);
if comp_len >= 1 && components[0] == "api2" { if comp_len >= 1 && components[0] == "api2" {
if comp_len >= 2 { if comp_len >= 2 {
let format = components[1]; let format = components[1];
let formatter = match format { let formatter = match format {
@ -725,8 +740,10 @@ async fn handle_request(
Ok(authid) => rpcenv.set_auth_id(Some(authid.to_string())), Ok(authid) => rpcenv.set_auth_id(Some(authid.to_string())),
Err(err) => { Err(err) => {
let peer = peer.ip(); let peer = peer.ip();
auth_logger()? auth_logger()?.log(format!(
.log(format!("authentication failure; rhost={} msg={}", peer, err)); "authentication failure; rhost={} msg={}",
peer, err
));
// always delay unauthorized calls by 3 seconds (from start of request) // always delay unauthorized calls by 3 seconds (from start of request)
let err = http_err!(UNAUTHORIZED, "authentication failed - {}", err); let err = http_err!(UNAUTHORIZED, "authentication failed - {}", err);
@ -743,7 +760,12 @@ async fn handle_request(
} }
Some(api_method) => { Some(api_method) => {
let auth_id = rpcenv.get_auth_id(); let auth_id = rpcenv.get_auth_id();
if !check_api_permission(api_method.access.permission, auth_id.as_deref(), &uri_param, user_info.as_ref()) { if !check_api_permission(
api_method.access.permission,
auth_id.as_deref(),
&uri_param,
user_info.as_ref(),
) {
let err = http_err!(FORBIDDEN, "permission check failed"); let err = http_err!(FORBIDDEN, "permission check failed");
tokio::time::sleep_until(Instant::from_std(access_forbidden_time)).await; tokio::time::sleep_until(Instant::from_std(access_forbidden_time)).await;
return Ok((formatter.format_error)(err)); return Ok((formatter.format_error)(err));
@ -752,7 +774,8 @@ async fn handle_request(
let result = if api_method.protected && env_type == RpcEnvironmentType::PUBLIC { let result = if api_method.protected && env_type == RpcEnvironmentType::PUBLIC {
proxy_protected_request(api_method, parts, body, peer).await proxy_protected_request(api_method, parts, body, peer).await
} else { } else {
handle_api_request(rpcenv, api_method, formatter, parts, body, uri_param).await handle_api_request(rpcenv, api_method, formatter, parts, body, uri_param)
.await
}; };
let mut response = match result { let mut response = match result {
@ -768,7 +791,6 @@ async fn handle_request(
return Ok(response); return Ok(response);
} }
} }
} }
} else { } else {
// not Auth required for accessing files! // not Auth required for accessing files!
@ -784,8 +806,14 @@ async fn handle_request(
Ok(auth_id) if !auth_id.is_token() => { Ok(auth_id) if !auth_id.is_token() => {
let userid = auth_id.user(); let userid = auth_id.user();
let new_csrf_token = assemble_csrf_prevention_token(csrf_secret(), userid); let new_csrf_token = assemble_csrf_prevention_token(csrf_secret(), userid);
return Ok(get_index(Some(userid.clone()), Some(new_csrf_token), language, &api, parts)); return Ok(get_index(
}, Some(userid.clone()),
Some(new_csrf_token),
language,
&api,
parts,
));
}
_ => { _ => {
tokio::time::sleep_until(Instant::from_std(delay_unauth_time)).await; tokio::time::sleep_until(Instant::from_std(delay_unauth_time)).await;
return Ok(get_index(None, None, language, &api, parts)); return Ok(get_index(None, None, language, &api, parts));