6 Commits

Author SHA1 Message Date
b4ed1fc1a3 Check redirect path and add trailing slash if necessary
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/tag Build is passing
2022-03-31 22:23:51 -04:00
aa8a187bda Mimic existing mirrors endpoint, add geoip
All checks were successful
continuous-integration/drone Build is passing
continuous-integration/drone/tag Build is passing
2022-03-31 22:04:19 -04:00
e06eced768 No more initial status messages = starting does nothing
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/tag Build is passing
2022-03-31 01:04:04 -04:00
6c52d1a5b2 Rewrite to add weighting and caching
All checks were successful
continuous-integration/drone/push Build is passing
2022-03-31 00:57:21 -04:00
13f95ee895 Allow real IP from loopback AND private
Some checks reported errors
continuous-integration/drone/push Build was killed
continuous-integration/drone/tag Build is passing
2022-03-30 22:58:15 -04:00
99c0137034 Fix redirect handler (again) to use notfound
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/tag Build is passing
2022-03-30 22:16:50 -04:00
9 changed files with 441 additions and 173 deletions

212
config.go Normal file
View File

@ -0,0 +1,212 @@
package main
import (
lru "github.com/hashicorp/golang-lru"
"github.com/oschwald/maxminddb-golang"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"net"
"net/url"
"strings"
"sync"
)
func reloadConfig() {
err := viper.ReadInConfig() // Find and read the config file
if err != nil { // Handle errors reading the config file
log.WithError(err).Fatalln("Unable to load config file")
}
// db will never be reloaded.
if db == nil {
// Load maxmind database
db, err = maxminddb.Open(viper.GetString("geodb"))
if err != nil {
log.WithError(err).Fatalln("Unable to open database")
}
}
// Refresh server cache if size changed
if serverCache == nil {
serverCache, err = lru.New(viper.GetInt("cacheSize"))
} else {
serverCache.Resize(viper.GetInt("cacheSize"))
}
// Set top choice count
topChoices = viper.GetInt("topChoices")
// Reload map file
reloadMap()
// Reload server list
reloadServers()
// Check top choices size
if topChoices > len(servers) {
topChoices = len(servers)
}
// Force check
go servers.Check()
}
func reloadServers() {
var serverList []ServerConfig
viper.UnmarshalKey("servers", &serverList)
var wg sync.WaitGroup
existing := make(map[string]int)
for i, server := range servers {
existing[server.Host] = i
}
hosts := make(map[string]bool)
for _, server := range serverList {
wg.Add(1)
var prefix string
if !strings.HasPrefix(server.Server, "http") {
prefix = "https://"
}
u, err := url.Parse(prefix + server.Server)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"server": server,
}).Warning("Server is invalid")
return
}
hosts[u.Host] = true
i := -1
if v, exists := existing[u.Host]; exists {
i = v
}
go func(i int, server ServerConfig, u *url.URL) {
defer wg.Done()
s := addServer(server, u)
if _, ok := existing[u.Host]; ok {
s.Redirects = servers[i].Redirects
servers[i] = s
} else {
s.Redirects = promauto.NewCounter(prometheus.CounterOpts{
Name: "armbian_router_redirects_" + metricReplacer.Replace(u.Host),
Help: "The number of redirects for server " + u.Host,
})
servers = append(servers, s)
log.WithFields(log.Fields{
"server": u.Host,
"path": u.Path,
"latitude": s.Latitude,
"longitude": s.Longitude,
}).Info("Added server")
}
}(i, server, u)
}
wg.Wait()
// Remove servers that no longer exist in the config
for i := len(servers) - 1; i >= 0; i-- {
if _, exists := hosts[servers[i].Host]; exists {
continue
}
log.WithFields(log.Fields{
"server": servers[i].Host,
}).Info("Removed server")
servers = append(servers[:i], servers[i+1:]...)
}
}
var metricReplacer = strings.NewReplacer(".", "_", "-", "_")
// addServer takes ServerConfig and constructs a server.
// This will create duplicate servers, but it will overwrite existing ones when changed.
func addServer(server ServerConfig, u *url.URL) *Server {
s := &Server{
Available: true,
Host: u.Host,
Path: u.Path,
Latitude: server.Latitude,
Longitude: server.Longitude,
Continent: server.Continent,
Weight: server.Weight,
}
// Defaults to 10 to allow servers to be set lower for lower priority
if s.Weight == 0 {
s.Weight = 10
}
ips, err := net.LookupIP(u.Host)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"server": s.Host,
}).Warning("Could not resolve address")
return nil
}
var city City
err = db.Lookup(ips[0], &city)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"server": s.Host,
"ip": ips[0],
}).Warning("Could not geolocate address")
return nil
}
if s.Continent == "" {
s.Continent = city.Continent.Code
}
if s.Latitude == 0 && s.Longitude == 0 {
s.Latitude = city.Location.Latitude
s.Longitude = city.Location.Longitude
}
return s
}
func reloadMap() {
mapFile := viper.GetString("dl_map")
if mapFile == "" {
return
}
log.WithField("file", mapFile).Info("Loading download map")
newMap, err := loadMap(mapFile)
if err != nil {
return
}
dlMap = newMap
}

View File

@ -1,16 +1,47 @@
geodb: GeoIP2-City.mmdb # GeoIP Database Path
geodb: GeoLite2-City.mmdb
dl_map: userdata.csv
# LRU Cache Size (in items)
cacheSize: 1024
# Server definition
# Weights are just like nginx, where if it's > 1 it'll be chosen x out of x + total times
# By default, the top 3 servers are used for choosing the best.
# server = full url or host+path
# weight = int
# optional: latitude, longitude (float)
servers: servers:
- https://mirrors.tuna.tsinghua.edu.cn/armbian-releases/ - server: armbian.12z.eu/apt/
- https://mirrors.bfsu.edu.cn/armbian-releases/ - server: armbian.chi.auroradev.org/apt/
- https://mirrors.nju.edu.cn/armbian-releases/ weight: 15
- https://mirrors.ustc.edu.cn/armbian-dl/ latitude: 41.8879
- https://mirror.12z.eu/pub/linux/armbian/dl/ longitude: -88.1995
- https://armbian.tnahosting.net/dl/ - server: armbian.hosthatch.com/apt/
- https://stpete-mirror.armbian.com/dl/ - server: armbian.lv.auroradev.org/apt/
- https://mirror.armbian.de/dl/ weight: 15
- https://mirrors.netix.net/armbian/dl/ - server: armbian.site-meganet.com/apt/
- https://mirrors.dotsrc.org/armbian-dl/ - server: armbian.systemonachip.net/apt/
- https://armbian.hosthatch.com/dl/ - server: armbian.tnahosting.net/apt/
- https://xogium.performanceservers.nl/dl/ weight: 15
- https://github.com/armbian/mirror/releases/download/ - server: au-mirror.bret.dk/armbian/apt/
- server: es-mirror.bret.dk/armbian/apt/
- server: imola.armbian.com/apt/
- server: mirror.iscas.ac.cn/armbian/
- server: mirror.sjtu.edu.cn/armbian/
- server: mirrors.aliyun.com/armbian/
continent: AS
- server: mirrors.bfsu.edu.cn/armbian/
- server: mirrors.dotsrc.org/armbian-apt/
weight: 15
- server: mirrors.netix.net/armbian/apt/
- server: mirrors.nju.edu.cn/armbian/
- server: mirrors.sustech.edu.cn/armbian/
- server: mirrors.tuna.tsinghua.edu.cn/armbian/
- server: mirrors.ustc.edu.cn/armbian/
- server: mirrors.xtom.de/armbian/
- server: sg-mirror.bret.dk/armbian/apt/
- server: stpete-mirror.armbian.com/apt/
- server: xogium.performanceservers.nl/apt/
- server: github.com/armbian/mirror/releases/download/
continent: GITHUB

2
go.mod
View File

@ -5,6 +5,8 @@ go 1.17
require ( require (
github.com/chi-middleware/logrus-logger v0.2.0 github.com/chi-middleware/logrus-logger v0.2.0
github.com/go-chi/chi/v5 v5.0.7 github.com/go-chi/chi/v5 v5.0.7
github.com/hashicorp/golang-lru v0.5.4
github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff
github.com/oschwald/maxminddb-golang v1.8.0 github.com/oschwald/maxminddb-golang v1.8.0
github.com/prometheus/client_golang v1.11.0 github.com/prometheus/client_golang v1.11.0
github.com/sirupsen/logrus v1.8.1 github.com/sirupsen/logrus v1.8.1

3
go.sum
View File

@ -224,6 +224,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@ -234,6 +235,8 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff h1:6NvhExg4omUC9NfA+l4Oq3ibNNeJUdiAF3iBVB0PlDk=
github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff/go.mod h1:ddfPX8Z28YMjiqoaJhNBzWHapTHXejnB5cDCUWDwriw=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=

58
http.go
View File

@ -12,11 +12,31 @@ import (
func statusHandler(w http.ResponseWriter, r *http.Request) { func statusHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
w.Write([]byte("OK"))
}
func legacyMirrorsHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
mirrors := make(map[string][]string)
for _, server := range servers {
u := &url.URL{
Scheme: r.URL.Scheme,
Host: server.Host,
Path: server.Path,
}
mirrors[server.Continent] = append(mirrors[server.Continent], u.String())
}
mirrors["default"] = append(mirrors["NA"], mirrors["EU"]...)
json.NewEncoder(w).Encode(mirrors)
} }
func mirrorsHandler(w http.ResponseWriter, r *http.Request) { func mirrorsHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(servers) json.NewEncoder(w).Encode(servers)
} }
@ -51,12 +71,22 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
redirectPath := path.Join(server.Path, r.URL.Path) redirectPath := path.Join(server.Path, r.URL.Path)
if dlMap != nil { if dlMap != nil {
if newPath, exists := dlMap[strings.TrimLeft(r.URL.Path, "/")]; exists { p := r.URL.Path
if p[0] != '/' {
p = "/" + p
}
if newPath, exists := dlMap[p]; exists {
downloadsMapped.Inc() downloadsMapped.Inc()
redirectPath = path.Join(server.Path, newPath) redirectPath = path.Join(server.Path, newPath)
} }
} }
if strings.HasSuffix(r.URL.Path, "/") && !strings.HasSuffix(redirectPath, "/") {
redirectPath += "/"
}
u := &url.URL{ u := &url.URL{
Scheme: scheme, Scheme: scheme,
Host: server.Host, Host: server.Host,
@ -72,7 +102,7 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
} }
func reloadHandler(w http.ResponseWriter, r *http.Request) { func reloadHandler(w http.ResponseWriter, r *http.Request) {
reloadMap() reloadConfig()
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
w.Write([]byte("OK")) w.Write([]byte("OK"))
@ -88,3 +118,25 @@ func dlMapHandler(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(dlMap) json.NewEncoder(w).Encode(dlMap)
} }
func geoIPHandler(w http.ResponseWriter, r *http.Request) {
ipStr, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
ip := net.ParseIP(ipStr)
var city City
err = db.Lookup(ip, &city)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(city)
}

176
main.go
View File

@ -4,19 +4,16 @@ import (
"flag" "flag"
"github.com/chi-middleware/logrus-logger" "github.com/chi-middleware/logrus-logger"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
lru "github.com/hashicorp/golang-lru"
"github.com/oschwald/maxminddb-golang" "github.com/oschwald/maxminddb-golang"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/viper" "github.com/spf13/viper"
"net"
"net/http" "net/http"
"net/url"
"os" "os"
"os/signal" "os/signal"
"strings"
"sync"
"syscall" "syscall"
) )
@ -35,16 +32,51 @@ var (
Name: "armbian_router_download_maps", Name: "armbian_router_download_maps",
Help: "The total number of mapped download paths", Help: "The total number of mapped download paths",
}) })
serverCache *lru.Cache
topChoices int
) )
// City represents a MaxmindDB city type LocationLookup struct {
type City struct {
Location struct { Location struct {
Latitude float64 `maxminddb:"latitude"` Latitude float64 `maxminddb:"latitude"`
Longitude float64 `maxminddb:"longitude"` Longitude float64 `maxminddb:"longitude"`
} `maxminddb:"location"` } `maxminddb:"location"`
} }
// City represents a MaxmindDB city
type City struct {
Continent struct {
Code string `maxminddb:"code" json:"code"`
GeoNameID uint `maxminddb:"geoname_id" json:"geoname_id"`
Names map[string]string `maxminddb:"names" json:"names"`
} `maxminddb:"continent" json:"continent"`
Country struct {
GeoNameID uint `maxminddb:"geoname_id" json:"geoname_id"`
IsoCode string `maxminddb:"iso_code" json:"iso_code"`
Names map[string]string `maxminddb:"names" json:"names"`
} `maxminddb:"country" json:"country"`
Location struct {
AccuracyRadius uint16 `maxminddb:"accuracy_radius" json:'accuracy_radius'`
Latitude float64 `maxminddb:"latitude" json:"latitude"`
Longitude float64 `maxminddb:"longitude" json:"longitude"`
} `maxminddb:"location"`
RegisteredCountry struct {
GeoNameID uint `maxminddb:"geoname_id" json:"geoname_id"`
IsoCode string `maxminddb:"iso_code" json:"iso_code"`
Names map[string]string `maxminddb:"names" json:"names"`
} `maxminddb:"registered_country" json:"registered_country"`
}
type ServerConfig struct {
Server string `mapstructure:"server" yaml:"server"`
Latitude float64 `mapstructure:"latitude" yaml:"latitude"`
Longitude float64 `mapstructure:"longitude" yaml:"longitude"`
Continent string `mapstructure:"continent"`
Weight int `mapstructure:"weight" yaml:"weight"`
}
var ( var (
configFlag = flag.String("config", "", "configuration file path") configFlag = flag.String("config", "", "configuration file path")
) )
@ -53,6 +85,8 @@ func main() {
flag.Parse() flag.Parse()
viper.SetDefault("bind", ":8080") viper.SetDefault("bind", ":8080")
viper.SetDefault("cacheSize", 1024)
viper.SetDefault("topChoices", 3)
viper.SetConfigName("dlrouter") // name of config file (without extension) viper.SetConfigName("dlrouter") // name of config file (without extension)
viper.SetConfigType("yaml") // REQUIRED if the config file does not have the extension in the name viper.SetConfigType("yaml") // REQUIRED if the config file does not have the extension in the name
@ -64,45 +98,7 @@ func main() {
viper.SetConfigFile(*configFlag) viper.SetConfigFile(*configFlag)
} }
err := viper.ReadInConfig() // Find and read the config file reloadConfig()
if err != nil { // Handle errors reading the config file
log.WithError(err).Fatalln("Unable to load config file")
}
db, err = maxminddb.Open(viper.GetString("geodb"))
if err != nil {
log.WithError(err).Fatalln("Unable to open database")
}
if mapFile := viper.GetString("dl_map"); mapFile != "" {
log.WithField("file", mapFile).Info("Loading download map")
dlMap, err = loadMap(mapFile)
if err != nil {
log.WithError(err).Fatalln("Unable to load download map")
}
}
serverList := viper.GetStringSlice("servers")
var wg sync.WaitGroup
for _, server := range serverList {
wg.Add(1)
go func(server string) {
defer wg.Done()
addServer(server)
}(server)
}
wg.Wait()
log.Info("Servers added, checking statuses")
// Start check loop // Start check loop
go servers.checkLoop() go servers.checkLoop()
@ -115,14 +111,19 @@ func main() {
r.Use(logger.Logger("router", log.StandardLogger())) r.Use(logger.Logger("router", log.StandardLogger()))
r.Get("/status", statusHandler) r.Get("/status", statusHandler)
r.Get("/mirrors", mirrorsHandler) r.Get("/mirrors", legacyMirrorsHandler)
r.Get("/mirrors.json", mirrorsHandler)
r.Post("/reload", reloadHandler) r.Post("/reload", reloadHandler)
r.Get("/dl_map", dlMapHandler) r.Get("/dl_map", dlMapHandler)
r.Get("/geoip", geoIPHandler)
r.Get("/metrics", promhttp.Handler().ServeHTTP) r.Get("/metrics", promhttp.Handler().ServeHTTP)
r.HandleFunc("/*", redirectHandler)
r.NotFound(redirectHandler)
go http.ListenAndServe(viper.GetString("bind"), r) go http.ListenAndServe(viper.GetString("bind"), r)
log.Info("Ready")
c := make(chan os.Signal) c := make(chan os.Signal)
signal.Notify(c, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGHUP) signal.Notify(c, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGHUP)
@ -134,85 +135,6 @@ func main() {
break break
} }
reloadMap() reloadConfig()
} }
} }
var metricReplacer = strings.NewReplacer(".", "_", "-", "_")
func addServer(server string) {
var prefix string
if !strings.HasPrefix(server, "http") {
prefix = "https://"
}
u, err := url.Parse(prefix + server)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"server": server,
}).Warning("Server is invalid")
return
}
ips, err := net.LookupIP(u.Host)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"server": server,
}).Warning("Could not resolve address")
return
}
var city City
err = db.Lookup(ips[0], &city)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"server": server,
}).Warning("Could not geolocate address")
return
}
redirects := promauto.NewCounter(prometheus.CounterOpts{
Name: "armbian_router_redirects_" + metricReplacer.Replace(u.Host),
Help: "The number of redirects for server " + u.Host,
})
servers = append(servers, &Server{
Host: u.Host,
Path: u.Path,
Latitude: city.Location.Latitude,
Longitude: city.Location.Longitude,
Redirects: redirects,
})
log.WithFields(log.Fields{
"server": u.Host,
"path": u.Path,
"latitude": city.Location.Latitude,
"longitude": city.Location.Longitude,
}).Info("Added server")
}
func reloadMap() {
mapFile := viper.GetString("dl_map")
if mapFile == "" {
return
}
log.WithField("file", mapFile).Info("Loading download map")
newMap, err := loadMap(mapFile)
if err != nil {
return
}
dlMap = newMap
}

3
map.go
View File

@ -4,7 +4,6 @@ import (
"encoding/csv" "encoding/csv"
"io" "io"
"os" "os"
"strings"
) )
func loadMap(file string) (map[string]string, error) { func loadMap(file string) (map[string]string, error) {
@ -33,7 +32,7 @@ func loadMap(file string) (map[string]string, error) {
return nil, err return nil, err
} }
m[strings.TrimLeft(row[0], "/")] = strings.TrimLeft(row[1], "/") m[row[0]] = row[1]
} }
return m, nil return m, nil

View File

@ -29,7 +29,9 @@ func RealIPMiddleware(f http.Handler) http.Handler {
return return
} }
if !net.ParseIP(host).IsPrivate() { netIP := net.ParseIP(host)
if !netIP.IsLoopback() && !netIP.IsPrivate() {
f.ServeHTTP(w, r) f.ServeHTTP(w, r)
return return
} }

View File

@ -1,12 +1,14 @@
package main package main
import ( import (
"github.com/jmcvetta/randutil"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"math" "math"
"net" "net"
"net/http" "net/http"
"runtime" "runtime"
"sort"
"strings" "strings"
"sync" "sync"
"time" "time"
@ -19,12 +21,14 @@ var (
) )
type Server struct { type Server struct {
Available bool Available bool `json:"available"`
Host string Host string `json:"host"`
Path string Path string `json:"path"`
Latitude float64 Latitude float64 `json:"latitude"`
Longitude float64 Longitude float64 `json:"longitude"`
Redirects prometheus.Counter Weight int `json:"weight"`
Continent string `json:"continent"`
Redirects prometheus.Counter `json:"-"`
} }
func (server *Server) checkStatus() { func (server *Server) checkStatus() {
@ -86,8 +90,8 @@ func (s ServerList) checkLoop() {
t := time.NewTicker(60 * time.Second) t := time.NewTicker(60 * time.Second)
for { for {
s.Check()
<-t.C <-t.C
s.Check()
} }
} }
@ -110,33 +114,74 @@ func (s ServerList) Check() {
wg.Wait() wg.Wait()
} }
// Closest will use GeoIP on the IP provided and find the closest server. // ComputedDistance is a wrapper that contains a Server and Distance.
type ComputedDistance struct {
Server *Server
Distance float64
}
// DistanceList is a list of Computed Distances with an easy "Choices" func
type DistanceList []ComputedDistance
func (d DistanceList) Choices() []randutil.Choice {
c := make([]randutil.Choice, len(d))
for i, item := range d {
c[i] = randutil.Choice{
Weight: item.Server.Weight,
Item: item,
}
}
return c
}
// Closest will use GeoIP on the IP provided and find the closest servers.
// When we have a list of x servers closest, we can choose a random or weighted one.
// Return values are the closest server, the distance, and if an error occurred. // Return values are the closest server, the distance, and if an error occurred.
func (s ServerList) Closest(ip net.IP) (*Server, float64, error) { func (s ServerList) Closest(ip net.IP) (*Server, float64, error) {
var city City choiceInterface, exists := serverCache.Get(ip.String())
err := db.Lookup(ip, &city)
if !exists {
var city LocationLookup
err := db.Lookup(ip, &city)
if err != nil {
return nil, -1, err
}
c := make(DistanceList, len(s))
for i, server := range s {
if !server.Available {
continue
}
c[i] = ComputedDistance{
Server: server,
Distance: Distance(city.Location.Latitude, city.Location.Longitude, server.Latitude, server.Longitude),
}
}
// Sort by distance
sort.Slice(s, func(i int, j int) bool {
return c[i].Distance < c[j].Distance
})
choiceInterface = c[0:topChoices].Choices()
serverCache.Add(ip.String(), choiceInterface)
}
choice, err := randutil.WeightedChoice(choiceInterface.([]randutil.Choice))
if err != nil { if err != nil {
return nil, -1, err return nil, -1, err
} }
var closest *Server dist := choice.Item.(ComputedDistance)
var closestDistance float64 = -1
for _, server := range s { return dist.Server, dist.Distance, nil
if !server.Available {
continue
}
distance := Distance(city.Location.Latitude, city.Location.Longitude, server.Latitude, server.Longitude)
if closestDistance == -1 || distance < closestDistance {
closestDistance = distance
closest = server
}
}
return closest, closestDistance, nil
} }
// haversin(θ) function // haversin(θ) function