No more initial status messages = starting does nothing

config.go

@@ -0,0 +1,206 @@
+package main
+
+import (
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/oschwald/maxminddb-golang"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	"net"
+	"net/url"
+	"strings"
+	"sync"
+)
+
+func reloadConfig() {
+	err := viper.ReadInConfig() // Find and read the config file
+
+	if err != nil { // Handle errors reading the config file
+		log.WithError(err).Fatalln("Unable to load config file")
+	}
+
+	// db will never be reloaded.
+	if db == nil {
+		// Load maxmind database
+		db, err = maxminddb.Open(viper.GetString("geodb"))
+
+		if err != nil {
+			log.WithError(err).Fatalln("Unable to open database")
+		}
+	}
+
+	// Refresh server cache if size changed
+	if serverCache == nil {
+		serverCache, err = lru.New(viper.GetInt("cacheSize"))
+	} else {
+		serverCache.Resize(viper.GetInt("cacheSize"))
+	}
+
+	// Set top choice count
+	topChoices = viper.GetInt("topChoices")
+
+	// Reload map file
+	reloadMap()
+
+	// Reload server list
+	reloadServers()
+
+	// Check top choices size
+	if topChoices > len(servers) {
+		topChoices = len(servers)
+	}
+
+	// Force check
+	go servers.Check()
+}
+
+func reloadServers() {
+	var serverList []ServerConfig
+	viper.UnmarshalKey("servers", &serverList)
+
+	var wg sync.WaitGroup
+
+	existing := make(map[string]int)
+
+	for i, server := range servers {
+		existing[server.Host] = i
+	}
+
+	hosts := make(map[string]bool)
+
+	for _, server := range serverList {
+		wg.Add(1)
+
+		var prefix string
+
+		if !strings.HasPrefix(server.Server, "http") {
+			prefix = "https://"
+		}
+
+		u, err := url.Parse(prefix + server.Server)
+
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error":  err,
+				"server": server,
+			}).Warning("Server is invalid")
+			return
+		}
+
+		hosts[u.Host] = true
+
+		i := -1
+
+		if v, exists := existing[u.Host]; exists {
+			i = v
+		}
+
+		go func(i int, server ServerConfig, u *url.URL) {
+			defer wg.Done()
+
+			s := addServer(server, u)
+
+			if _, ok := existing[u.Host]; ok {
+				s.Redirects = servers[i].Redirects
+
+				servers[i] = s
+			} else {
+				s.Redirects = promauto.NewCounter(prometheus.CounterOpts{
+					Name: "armbian_router_redirects_" + metricReplacer.Replace(u.Host),
+					Help: "The number of redirects for server " + u.Host,
+				})
+
+				servers = append(servers, s)
+
+				log.WithFields(log.Fields{
+					"server":    u.Host,
+					"path":      u.Path,
+					"latitude":  s.Latitude,
+					"longitude": s.Longitude,
+				}).Info("Added server")
+			}
+		}(i, server, u)
+	}
+
+	wg.Wait()
+
+	// Remove servers that no longer exist in the config
+	for i := len(servers) - 1; i >= 0; i-- {
+		if _, exists := hosts[servers[i].Host]; exists {
+			continue
+		}
+
+		log.WithFields(log.Fields{
+			"server": servers[i].Host,
+		}).Info("Removed server")
+
+		servers = append(servers[:i], servers[i+1:]...)
+	}
+}
+
+var metricReplacer = strings.NewReplacer(".", "_", "-", "_")
+
+// addServer takes ServerConfig and constructs a server.
+// This will create duplicate servers, but it will overwrite existing ones when changed.
+func addServer(server ServerConfig, u *url.URL) *Server {
+	s := &Server{
+		Available: true,
+		Host:      u.Host,
+		Path:      u.Path,
+		Latitude:  server.Latitude,
+		Longitude: server.Longitude,
+		Weight:    server.Weight,
+	}
+
+	if s.Weight == 0 {
+		s.Weight = 1
+	}
+
+	if s.Latitude == 0 && s.Longitude == 0 {
+		ips, err := net.LookupIP(u.Host)
+
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error":  err,
+				"server": s.Host,
+			}).Warning("Could not resolve address")
+			return nil
+		}
+
+		var city City
+		err = db.Lookup(ips[0], &city)
+
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error":  err,
+				"server": s.Host,
+				"ip":     ips[0],
+			}).Warning("Could not geolocate address")
+			return nil
+		}
+
+		s.Latitude = city.Location.Latitude
+		s.Longitude = city.Location.Longitude
+	}
+
+	return s
+}
+
+func reloadMap() {
+	mapFile := viper.GetString("dl_map")
+
+	if mapFile == "" {
+		return
+	}
+
+	log.WithField("file", mapFile).Info("Loading download map")
+
+	newMap, err := loadMap(mapFile)
+
+	if err != nil {
+		return
+	}
+
+	dlMap = newMap
+}

dlrouter.yaml

@@ -1,16 +1,43 @@
-geodb: GeoIP2-City.mmdb
+# GeoIP Database Path
+geodb: GeoLite2-City.mmdb
 
+# LRU Cache Size (in items)
+cacheSize: 1024
+
+# Server definition
+# Weights are just like nginx, where if it's > 1 it'll be chosen x out of x + total times
+# By default, the top 3 servers are used for choosing the best.
+# server = full url or host+path
+# weight = int
+# optional: latitude, longitude (float)
 servers:
-    - https://mirrors.tuna.tsinghua.edu.cn/armbian-releases/
-    - https://mirrors.bfsu.edu.cn/armbian-releases/
-    - https://mirrors.nju.edu.cn/armbian-releases/
-    - https://mirrors.ustc.edu.cn/armbian-dl/
-    - https://mirror.12z.eu/pub/linux/armbian/dl/
-    - https://armbian.tnahosting.net/dl/
-    - https://stpete-mirror.armbian.com/dl/
-    - https://mirror.armbian.de/dl/
-    - https://mirrors.netix.net/armbian/dl/
-    - https://mirrors.dotsrc.org/armbian-dl/
-    - https://armbian.hosthatch.com/dl/
-    - https://xogium.performanceservers.nl/dl/
-    - https://github.com/armbian/mirror/releases/download/
+    - server: armbian.12z.eu/apt/
+    - server: armbian.chi.auroradev.org/apt/
+      weight: 5
+      latitude: 41.8879
+      longitude: -88.1995
+    - server: armbian.hosthatch.com/apt/
+    - server: armbian.lv.auroradev.org/apt/
+      weight: 5
+    - server: armbian.site-meganet.com/apt/
+    - server: armbian.systemonachip.net/apt/
+    - server: armbian.tnahosting.net/apt/
+      weight: 5
+    - server: au-mirror.bret.dk/armbian/apt/
+    - server: es-mirror.bret.dk/armbian/apt/
+    - server: imola.armbian.com/apt/
+    - server: mirror.iscas.ac.cn/armbian/
+    - server: mirror.sjtu.edu.cn/armbian/
+    - server: mirrors.aliyun.com/armbian/
+    - server: mirrors.bfsu.edu.cn/armbian/
+    - server: mirrors.dotsrc.org/armbian-apt/
+      weight: 5
+    - server: mirrors.netix.net/armbian/apt/
+    - server: mirrors.nju.edu.cn/armbian/
+    - server: mirrors.sustech.edu.cn/armbian/
+    - server: mirrors.tuna.tsinghua.edu.cn/armbian/
+    - server: mirrors.ustc.edu.cn/armbian/
+    - server: mirrors.xtom.de/armbian/
+    - server: sg-mirror.bret.dk/armbian/apt/
+    - server: stpete-mirror.armbian.com/apt/
+    - server: xogium.performanceservers.nl/apt/

go.mod

@@ -5,6 +5,8 @@ go 1.17
 require (
 	github.com/chi-middleware/logrus-logger v0.2.0
 	github.com/go-chi/chi/v5 v5.0.7
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff
 	github.com/oschwald/maxminddb-golang v1.8.0
 	github.com/prometheus/client_golang v1.11.0
 	github.com/sirupsen/logrus v1.8.1

go.sum

@@ -224,6 +224,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@@ -234,6 +235,8 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff h1:6NvhExg4omUC9NfA+l4Oq3ibNNeJUdiAF3iBVB0PlDk=
+github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff/go.mod h1:ddfPX8Z28YMjiqoaJhNBzWHapTHXejnB5cDCUWDwriw=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=

http.go

@@ -3,8 +3,6 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/viper"
 	"net"
 	"net/http"
 	"net/url"
@@ -74,22 +72,10 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func reloadHandler(w http.ResponseWriter, r *http.Request) {
-	if mapFile := viper.GetString("dl_map"); mapFile != "" {
-		log.WithField("file", mapFile).Info("Loading download map")
+	reloadMap()
 
-		newMap, err := loadMap(mapFile)
-
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		dlMap = newMap
-
-		return
-	}
-
-	w.WriteHeader(http.StatusNotFound)
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("OK"))
 }
 
 func dlMapHandler(w http.ResponseWriter, r *http.Request) {

main.go

@@ -1,19 +1,20 @@
 package main
 
 import (
+	"flag"
 	"github.com/chi-middleware/logrus-logger"
 	"github.com/go-chi/chi/v5"
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/oschwald/maxminddb-golang"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
-	"net"
 	"net/http"
-	"net/url"
-	"strings"
-	"sync"
+	"os"
+	"os/signal"
+	"syscall"
 )
 
 var (
@@ -31,6 +32,10 @@ var (
 		Name: "armbian_router_download_maps",
 		Help: "The total number of mapped download paths",
 	})
+
+	serverCache *lru.Cache
+
+	topChoices int
 )
 
 // City represents a MaxmindDB city
@@ -41,53 +46,35 @@ type City struct {
 	} `maxminddb:"location"`
 }
 
+type ServerConfig struct {
+	Server    string  `mapstructure:"server" yaml:"server"`
+	Latitude  float64 `mapstructure:"latitude" yaml:"latitude"`
+	Longitude float64 `mapstructure:"longitude" yaml:"longitude"`
+	Weight    int     `mapstructure:"weight" yaml:"weight"`
+}
+
+var (
+	configFlag = flag.String("config", "", "configuration file path")
+)
+
 func main() {
+	flag.Parse()
+
 	viper.SetDefault("bind", ":8080")
+	viper.SetDefault("cacheSize", 1024)
+	viper.SetDefault("topChoices", 3)
 
 	viper.SetConfigName("dlrouter")        // name of config file (without extension)
 	viper.SetConfigType("yaml")            // REQUIRED if the config file does not have the extension in the name
 	viper.AddConfigPath("/etc/dlrouter/")  // path to look for the config file in
 	viper.AddConfigPath("$HOME/.dlrouter") // call multiple times to add many search paths
 	viper.AddConfigPath(".")               // optionally look for config in the working directory
-	err := viper.ReadInConfig()            // Find and read the config file
 
-	if err != nil { // Handle errors reading the config file
-		log.WithError(err).Fatalln("Unable to load config file")
+	if *configFlag != "" {
+		viper.SetConfigFile(*configFlag)
 	}
 
-	db, err = maxminddb.Open(viper.GetString("geodb"))
-
-	if err != nil {
-		log.WithError(err).Fatalln("Unable to open database")
-	}
-
-	if mapFile := viper.GetString("dl_map"); mapFile != "" {
-		log.WithField("file", mapFile).Info("Loading download map")
-
-		dlMap, err = loadMap(mapFile)
-
-		if err != nil {
-			log.WithError(err).Fatalln("Unable to load download map")
-		}
-	}
-
-	serverList := viper.GetStringSlice("servers")
-
-	var wg sync.WaitGroup
-
-	for _, server := range serverList {
-		wg.Add(1)
-
-		go func(server string) {
-			defer wg.Done()
-
-			addServer(server)
-		}(server)
-	}
-
-	wg.Wait()
-
-	log.Info("Servers added, checking statuses")
+	reloadConfig()
 
 	// Start check loop
 	go servers.checkLoop()
@@ -99,73 +86,29 @@ func main() {
 	r.Use(RealIPMiddleware)
 	r.Use(logger.Logger("router", log.StandardLogger()))
 
-	r.HandleFunc("/status", statusHandler)
-	r.HandleFunc("/mirrors", mirrorsHandler)
-	r.HandleFunc("/reload", reloadHandler)
-	r.HandleFunc("/dl_map", dlMapHandler)
-	r.Handle("/metrics", promhttp.Handler())
-	r.HandleFunc("/", redirectHandler)
+	r.Get("/status", statusHandler)
+	r.Get("/mirrors", mirrorsHandler)
+	r.Post("/reload", reloadHandler)
+	r.Get("/dl_map", dlMapHandler)
+	r.Get("/metrics", promhttp.Handler().ServeHTTP)
 
-	http.ListenAndServe(viper.GetString("bind"), r)
-}
-
-var metricReplacer = strings.NewReplacer(".", "_", "-", "_")
-
-func addServer(server string) {
-	var prefix string
-
-	if !strings.HasPrefix(server, "http") {
-		prefix = "https://"
-	}
-
-	u, err := url.Parse(prefix + server)
-
-	if err != nil {
-		log.WithFields(log.Fields{
-			"error":  err,
-			"server": server,
-		}).Warning("Server is invalid")
-		return
-	}
-
-	ips, err := net.LookupIP(u.Host)
-
-	if err != nil {
-		log.WithFields(log.Fields{
-			"error":  err,
-			"server": server,
-		}).Warning("Could not resolve address")
-		return
-	}
-
-	var city City
-	err = db.Lookup(ips[0], &city)
-
-	if err != nil {
-		log.WithFields(log.Fields{
-			"error":  err,
-			"server": server,
-		}).Warning("Could not geolocate address")
-		return
-	}
-
-	redirects := promauto.NewCounter(prometheus.CounterOpts{
-		Name: "armbian_router_redirects_" + metricReplacer.Replace(u.Host),
-		Help: "The number of redirects for server " + u.Host,
-	})
-
-	servers = append(servers, &Server{
-		Host:      u.Host,
-		Path:      u.Path,
-		Latitude:  city.Location.Latitude,
-		Longitude: city.Location.Longitude,
-		Redirects: redirects,
-	})
-
-	log.WithFields(log.Fields{
-		"server":    u.Host,
-		"path":      u.Path,
-		"latitude":  city.Location.Latitude,
-		"longitude": city.Location.Longitude,
-	}).Info("Added server")
+	r.NotFound(redirectHandler)
+
+	go http.ListenAndServe(viper.GetString("bind"), r)
+
+	log.Info("Ready")
+
+	c := make(chan os.Signal)
+
+	signal.Notify(c, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGHUP)
+
+	for {
+		sig := <-c
+
+		if sig != syscall.SIGHUP {
+			break
+		}
+
+		reloadConfig()
+	}
 }

middleware.go

@@ -29,7 +29,9 @@ func RealIPMiddleware(f http.Handler) http.Handler {
 			return
 		}
 
-		if !net.ParseIP(host).IsPrivate() {
+		netIP := net.ParseIP(host)
+
+		if !netIP.IsLoopback() && !netIP.IsPrivate() {
 			f.ServeHTTP(w, r)
 			return
 		}

servers.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"github.com/jmcvetta/randutil"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"math"
 	"net"
 	"net/http"
 	"runtime"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -24,6 +26,7 @@ type Server struct {
 	Path      string
 	Latitude  float64
 	Longitude float64
+	Weight    int
 	Redirects prometheus.Counter
 }
 
@@ -86,8 +89,8 @@ func (s ServerList) checkLoop() {
 	t := time.NewTicker(60 * time.Second)
 
 	for {
-		s.Check()
 		<-t.C
+		s.Check()
 	}
 }
 
@@ -110,9 +113,38 @@ func (s ServerList) Check() {
 	wg.Wait()
 }
 
-// Closest will use GeoIP on the IP provided and find the closest server.
+// ComputedDistance is a wrapper that contains a Server and Distance.
+type ComputedDistance struct {
+	Server   *Server
+	Distance float64
+}
+
+// DistanceList is a list of Computed Distances with an easy "Choices" func
+type DistanceList []ComputedDistance
+
+func (d DistanceList) Choices() []randutil.Choice {
+	c := make([]randutil.Choice, len(d))
+
+	for i, item := range d {
+		c[i] = randutil.Choice{
+			Weight: item.Server.Weight,
+			Item:   item,
+		}
+	}
+
+	return c
+}
+
+// Closest will use GeoIP on the IP provided and find the closest servers.
+// When we have a list of x servers closest, we can choose a random or weighted one.
 // Return values are the closest server, the distance, and if an error occurred.
 func (s ServerList) Closest(ip net.IP) (*Server, float64, error) {
+	i, exists := serverCache.Get(ip.String())
+
+	if exists {
+		return i.(ComputedDistance).Server, i.(ComputedDistance).Distance, nil
+	}
+
 	var city City
 	err := db.Lookup(ip, &city)
 
@@ -120,23 +152,35 @@ func (s ServerList) Closest(ip net.IP) (*Server, float64, error) {
 		return nil, -1, err
 	}
 
-	var closest *Server
-	var closestDistance float64 = -1
+	c := make(DistanceList, len(s))
 
-	for _, server := range s {
+	for i, server := range s {
 		if !server.Available {
 			continue
 		}
 
-		distance := Distance(city.Location.Latitude, city.Location.Longitude, server.Latitude, server.Longitude)
-
-		if closestDistance == -1 || distance < closestDistance {
-			closestDistance = distance
-			closest = server
+		c[i] = ComputedDistance{
+			Server:   server,
+			Distance: Distance(city.Location.Latitude, city.Location.Longitude, server.Latitude, server.Longitude),
 		}
 	}
 
-	return closest, closestDistance, nil
+	// Sort by distance
+	sort.Slice(s, func(i int, j int) bool {
+		return c[i].Distance < c[j].Distance
+	})
+
+	choice, err := randutil.WeightedChoice(c[0:topChoices].Choices())
+
+	if err != nil {
+		return nil, -1, err
+	}
+
+	dist := choice.Item.(ComputedDistance)
+
+	serverCache.Add(ip.String(), dist)
+
+	return dist.Server, dist.Distance, nil
 }
 
 // haversin(θ) function