Add mirror status endpoint, timestamp on last activity

assets/status-down.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="82" height="20" role="img" aria-label="status: down"><title>status: down</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="82" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="43" height="20" fill="#555"/><rect x="43" width="39" height="20" fill="#e05d44"/><rect width="82" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="225" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="330">status</text><text x="225" y="140" transform="scale(.1)" fill="#fff" textLength="330">status</text><text aria-hidden="true" x="615" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="290">down</text><text x="615" y="140" transform="scale(.1)" fill="#fff" textLength="290">down</text></g></svg>

assets/status-unknown.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="104" height="20" role="img" aria-label="status: unknown"><title>status: unknown</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="104" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="43" height="20" fill="#555"/><rect x="43" width="61" height="20" fill="#9f9f9f"/><rect width="104" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="225" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="330">status</text><text x="225" y="140" transform="scale(.1)" fill="#fff" textLength="330">status</text><text aria-hidden="true" x="725" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="510">unknown</text><text x="725" y="140" transform="scale(.1)" fill="#fff" textLength="510">unknown</text></g></svg>

assets/status-up.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="66" height="20" role="img" aria-label="status: up"><title>status: up</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="66" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="43" height="20" fill="#555"/><rect x="43" width="23" height="20" fill="#4c1"/><rect width="66" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="225" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="330">status</text><text x="225" y="140" transform="scale(.1)" fill="#fff" textLength="330">status</text><text aria-hidden="true" x="535" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="130">up</text><text x="535" y="140" transform="scale(.1)" fill="#fff" textLength="130">up</text></g></svg>

config.go

@@ -0,0 +1,233 @@
+package main
+
+import (
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/oschwald/maxminddb-golang"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	"net"
+	"net/url"
+	"strings"
+	"sync"
+)
+
+func reloadConfig() {
+	log.Info("Loading configuration...")
+
+	err := viper.ReadInConfig() // Find and read the config file
+
+	if err != nil { // Handle errors reading the config file
+		log.WithError(err).Fatalln("Unable to load config file")
+	}
+
+	// db will never be reloaded.
+	if db == nil {
+		// Load maxmind database
+		db, err = maxminddb.Open(viper.GetString("geodb"))
+
+		if err != nil {
+			log.WithError(err).Fatalln("Unable to open database")
+		}
+	}
+
+	// Refresh server cache if size changed
+	if serverCache == nil {
+		serverCache, err = lru.New(viper.GetInt("cacheSize"))
+	} else {
+		serverCache.Resize(viper.GetInt("cacheSize"))
+	}
+
+	// Set top choice count
+	topChoices = viper.GetInt("topChoices")
+
+	// Reload map file
+	reloadMap()
+
+	// Reload server list
+	reloadServers()
+
+	// Create mirror map
+	mirrors := make(map[string][]*Server)
+
+	for _, server := range servers {
+		mirrors[server.Continent] = append(mirrors[server.Continent], server)
+	}
+
+	mirrors["default"] = append(mirrors["NA"], mirrors["EU"]...)
+
+	regionMap = mirrors
+
+	hosts := make(map[string]*Server)
+
+	for _, server := range servers {
+		hosts[server.Host] = server
+	}
+
+	hostMap = hosts
+
+	// Check top choices size
+	if topChoices > len(servers) {
+		topChoices = len(servers)
+	}
+
+	// Force check
+	go servers.Check()
+}
+
+func reloadServers() {
+	var serverList []ServerConfig
+	viper.UnmarshalKey("servers", &serverList)
+
+	var wg sync.WaitGroup
+
+	existing := make(map[string]int)
+
+	for i, server := range servers {
+		existing[server.Host] = i
+	}
+
+	hosts := make(map[string]bool)
+
+	for _, server := range serverList {
+		wg.Add(1)
+
+		var prefix string
+
+		if !strings.HasPrefix(server.Server, "http") {
+			prefix = "https://"
+		}
+
+		u, err := url.Parse(prefix + server.Server)
+
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error":  err,
+				"server": server,
+			}).Warning("Server is invalid")
+			return
+		}
+
+		hosts[u.Host] = true
+
+		i := -1
+
+		if v, exists := existing[u.Host]; exists {
+			i = v
+		}
+
+		go func(i int, server ServerConfig, u *url.URL) {
+			defer wg.Done()
+
+			s := addServer(server, u)
+
+			if _, ok := existing[u.Host]; ok {
+				s.Redirects = servers[i].Redirects
+
+				servers[i] = s
+			} else {
+				s.Redirects = promauto.NewCounter(prometheus.CounterOpts{
+					Name: "armbian_router_redirects_" + metricReplacer.Replace(u.Host),
+					Help: "The number of redirects for server " + u.Host,
+				})
+
+				servers = append(servers, s)
+
+				log.WithFields(log.Fields{
+					"server":    u.Host,
+					"path":      u.Path,
+					"latitude":  s.Latitude,
+					"longitude": s.Longitude,
+				}).Info("Added server")
+			}
+		}(i, server, u)
+	}
+
+	wg.Wait()
+
+	// Remove servers that no longer exist in the config
+	for i := len(servers) - 1; i >= 0; i-- {
+		if _, exists := hosts[servers[i].Host]; exists {
+			continue
+		}
+
+		log.WithFields(log.Fields{
+			"server": servers[i].Host,
+		}).Info("Removed server")
+
+		servers = append(servers[:i], servers[i+1:]...)
+	}
+}
+
+var metricReplacer = strings.NewReplacer(".", "_", "-", "_")
+
+// addServer takes ServerConfig and constructs a server.
+// This will create duplicate servers, but it will overwrite existing ones when changed.
+func addServer(server ServerConfig, u *url.URL) *Server {
+	s := &Server{
+		Available: true,
+		Host:      u.Host,
+		Path:      u.Path,
+		Latitude:  server.Latitude,
+		Longitude: server.Longitude,
+		Continent: server.Continent,
+		Weight:    server.Weight,
+	}
+
+	// Defaults to 10 to allow servers to be set lower for lower priority
+	if s.Weight == 0 {
+		s.Weight = 10
+	}
+
+	ips, err := net.LookupIP(u.Host)
+
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":  err,
+			"server": s.Host,
+		}).Warning("Could not resolve address")
+		return nil
+	}
+
+	var city City
+	err = db.Lookup(ips[0], &city)
+
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":  err,
+			"server": s.Host,
+			"ip":     ips[0],
+		}).Warning("Could not geolocate address")
+		return nil
+	}
+
+	if s.Continent == "" {
+		s.Continent = city.Continent.Code
+	}
+
+	if s.Latitude == 0 && s.Longitude == 0 {
+		s.Latitude = city.Location.Latitude
+		s.Longitude = city.Location.Longitude
+	}
+
+	return s
+}
+
+func reloadMap() {
+	mapFile := viper.GetString("dl_map")
+
+	if mapFile == "" {
+		return
+	}
+
+	log.WithField("file", mapFile).Info("Loading download map")
+
+	newMap, err := loadMap(mapFile)
+
+	if err != nil {
+		return
+	}
+
+	dlMap = newMap
+}

dlrouter.yaml

@@ -1,16 +1,47 @@
-geodb: GeoIP2-City.mmdb
+# GeoIP Database Path
+geodb: GeoLite2-City.mmdb
+dl_map: userdata.csv
 
+# LRU Cache Size (in items)
+cacheSize: 1024
+
+# Server definition
+# Weights are just like nginx, where if it's > 1 it'll be chosen x out of x + total times
+# By default, the top 3 servers are used for choosing the best.
+# server = full url or host+path
+# weight = int
+# optional: latitude, longitude (float)
 servers:
-    - https://mirrors.tuna.tsinghua.edu.cn/armbian-releases/
-    - https://mirrors.bfsu.edu.cn/armbian-releases/
-    - https://mirrors.nju.edu.cn/armbian-releases/
-    - https://mirrors.ustc.edu.cn/armbian-dl/
-    - https://mirror.12z.eu/pub/linux/armbian/dl/
-    - https://armbian.tnahosting.net/dl/
-    - https://stpete-mirror.armbian.com/dl/
-    - https://mirror.armbian.de/dl/
-    - https://mirrors.netix.net/armbian/dl/
-    - https://mirrors.dotsrc.org/armbian-dl/
-    - https://armbian.hosthatch.com/dl/
-    - https://xogium.performanceservers.nl/dl/
-    - https://github.com/armbian/mirror/releases/download/
+    - server: armbian.12z.eu/apt/
+    - server: armbian.chi.auroradev.org/apt/
+      weight: 15
+      latitude: 41.8879
+      longitude: -88.1995
+    - server: armbian.hosthatch.com/apt/
+    - server: armbian.lv.auroradev.org/apt/
+      weight: 15
+    - server: armbian.site-meganet.com/apt/
+    - server: armbian.systemonachip.net/apt/
+    - server: armbian.tnahosting.net/apt/
+      weight: 15
+    - server: au-mirror.bret.dk/armbian/apt/
+    - server: es-mirror.bret.dk/armbian/apt/
+    - server: imola.armbian.com/apt/
+    - server: mirror.iscas.ac.cn/armbian/
+    - server: mirror.sjtu.edu.cn/armbian/
+    - server: mirrors.aliyun.com/armbian/
+      continent: AS
+    - server: mirrors.bfsu.edu.cn/armbian/
+    - server: mirrors.dotsrc.org/armbian-apt/
+      weight: 15
+    - server: mirrors.netix.net/armbian/apt/
+    - server: mirrors.nju.edu.cn/armbian/
+    - server: mirrors.sustech.edu.cn/armbian/
+    - server: mirrors.tuna.tsinghua.edu.cn/armbian/
+    - server: mirrors.ustc.edu.cn/armbian/
+    - server: mirrors.xtom.de/armbian/
+    - server: sg-mirror.bret.dk/armbian/apt/
+    - server: stpete-mirror.armbian.com/apt/
+    - server: xogium.performanceservers.nl/apt/
+    - server: github.com/armbian/mirror/releases/download/
+      continent: GITHUB

go.mod

@@ -5,6 +5,8 @@ go 1.17
 require (
 	github.com/chi-middleware/logrus-logger v0.2.0
 	github.com/go-chi/chi/v5 v5.0.7
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff
 	github.com/oschwald/maxminddb-golang v1.8.0
 	github.com/prometheus/client_golang v1.11.0
 	github.com/sirupsen/logrus v1.8.1

go.sum

@@ -224,6 +224,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@@ -234,6 +235,8 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff h1:6NvhExg4omUC9NfA+l4Oq3ibNNeJUdiAF3iBVB0PlDk=
+github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff/go.mod h1:ddfPX8Z28YMjiqoaJhNBzWHapTHXejnB5cDCUWDwriw=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=

http.go

@@ -3,21 +3,18 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/jmcvetta/randutil"
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strings"
 )
 
 func statusHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
-}
-
-func mirrorsHandler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	json.NewEncoder(w).Encode(servers)
+	w.Write([]byte("OK"))
 }
 
 func redirectHandler(w http.ResponseWriter, r *http.Request) {
@@ -30,16 +27,57 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 
 	ip := net.ParseIP(ipStr)
 
-	// TODO: This is temporary to allow testing on private addresses.
-	if ip.IsPrivate() {
-		ip = net.ParseIP("1.1.1.1")
+	if ip.IsLoopback() || ip.IsPrivate() {
+		overrideIP := os.Getenv("OVERRIDE_IP")
+
+		if overrideIP == "" {
+			overrideIP = "1.1.1.1"
+		}
+
+		ip = net.ParseIP(overrideIP)
 	}
 
-	server, distance, err := servers.Closest(ip)
+	var server *Server
+	var distance float64
 
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
+	if strings.HasPrefix(r.URL.Path, "/region") {
+		parts := strings.Split(r.URL.Path, "/")
+
+		// region = parts[2]
+		if mirrors, ok := regionMap[parts[2]]; ok {
+			choices := make([]randutil.Choice, len(mirrors))
+
+			for i, item := range mirrors {
+				if !item.Available {
+					continue
+				}
+
+				choices[i] = randutil.Choice{
+					Weight: item.Weight,
+					Item:   item,
+				}
+			}
+
+			choice, err := randutil.WeightedChoice(choices)
+
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+
+			server = choice.Item.(*Server)
+
+			r.URL.Path = strings.Join(parts[3:], "/")
+		}
+	}
+
+	if server == nil {
+		server, distance, err = servers.Closest(ip)
+
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
 	}
 
 	scheme := r.URL.Scheme
@@ -51,12 +89,22 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 	redirectPath := path.Join(server.Path, r.URL.Path)
 
 	if dlMap != nil {
-		if newPath, exists := dlMap[strings.TrimLeft(r.URL.Path, "/")]; exists {
+		p := r.URL.Path
+
+		if p[0] != '/' {
+			p = "/" + p
+		}
+
+		if newPath, exists := dlMap[p]; exists {
 			downloadsMapped.Inc()
 			redirectPath = path.Join(server.Path, newPath)
 		}
 	}
 
+	if strings.HasSuffix(r.URL.Path, "/") && !strings.HasSuffix(redirectPath, "/") {
+		redirectPath += "/"
+	}
+
 	u := &url.URL{
 		Scheme: scheme,
 		Host:   server.Host,
@@ -66,13 +114,16 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 	server.Redirects.Inc()
 	redirectsServed.Inc()
 
-	w.Header().Set("X-Geo-Distance", fmt.Sprintf("%f", distance))
+	if distance > 0 {
+		w.Header().Set("X-Geo-Distance", fmt.Sprintf("%f", distance))
+	}
+
 	w.Header().Set("Location", u.String())
 	w.WriteHeader(http.StatusFound)
 }
 
 func reloadHandler(w http.ResponseWriter, r *http.Request) {
-	reloadMap()
+	reloadConfig()
 
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte("OK"))
@@ -88,3 +139,25 @@ func dlMapHandler(w http.ResponseWriter, r *http.Request) {
 
 	json.NewEncoder(w).Encode(dlMap)
 }
+
+func geoIPHandler(w http.ResponseWriter, r *http.Request) {
+	ipStr, _, err := net.SplitHostPort(r.RemoteAddr)
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	ip := net.ParseIP(ipStr)
+
+	var city City
+	err = db.Lookup(ip, &city)
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(city)
+}

main.go

@@ -4,27 +4,26 @@ import (
 	"flag"
 	"github.com/chi-middleware/logrus-logger"
 	"github.com/go-chi/chi/v5"
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/oschwald/maxminddb-golang"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
-	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"os/signal"
-	"strings"
-	"sync"
 	"syscall"
 )
 
 var (
-	db      *maxminddb.Reader
-	servers ServerList
-
-	dlMap map[string]string
+	db         *maxminddb.Reader
+	servers    ServerList
+	regionMap  map[string][]*Server
+	hostMap    map[string]*Server
+	dlMap      map[string]string
+	topChoices int
 
 	redirectsServed = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "armbian_router_redirects",
@@ -35,16 +34,49 @@ var (
 		Name: "armbian_router_download_maps",
 		Help: "The total number of mapped download paths",
 	})
+
+	serverCache *lru.Cache
 )
 
-// City represents a MaxmindDB city
-type City struct {
+type LocationLookup struct {
 	Location struct {
 		Latitude  float64 `maxminddb:"latitude"`
 		Longitude float64 `maxminddb:"longitude"`
 	} `maxminddb:"location"`
 }
 
+// City represents a MaxmindDB city
+type City struct {
+	Continent struct {
+		Code      string            `maxminddb:"code" json:"code"`
+		GeoNameID uint              `maxminddb:"geoname_id" json:"geoname_id"`
+		Names     map[string]string `maxminddb:"names" json:"names"`
+	} `maxminddb:"continent" json:"continent"`
+	Country struct {
+		GeoNameID uint              `maxminddb:"geoname_id" json:"geoname_id"`
+		IsoCode   string            `maxminddb:"iso_code" json:"iso_code"`
+		Names     map[string]string `maxminddb:"names" json:"names"`
+	} `maxminddb:"country" json:"country"`
+	Location struct {
+		AccuracyRadius uint16  `maxminddb:"accuracy_radius" json:'accuracy_radius'`
+		Latitude       float64 `maxminddb:"latitude" json:"latitude"`
+		Longitude      float64 `maxminddb:"longitude" json:"longitude"`
+	} `maxminddb:"location"`
+	RegisteredCountry struct {
+		GeoNameID uint              `maxminddb:"geoname_id" json:"geoname_id"`
+		IsoCode   string            `maxminddb:"iso_code" json:"iso_code"`
+		Names     map[string]string `maxminddb:"names" json:"names"`
+	} `maxminddb:"registered_country" json:"registered_country"`
+}
+
+type ServerConfig struct {
+	Server    string  `mapstructure:"server" yaml:"server"`
+	Latitude  float64 `mapstructure:"latitude" yaml:"latitude"`
+	Longitude float64 `mapstructure:"longitude" yaml:"longitude"`
+	Continent string  `mapstructure:"continent"`
+	Weight    int     `mapstructure:"weight" yaml:"weight"`
+}
+
 var (
 	configFlag = flag.String("config", "", "configuration file path")
 )
@@ -53,6 +85,8 @@ func main() {
 	flag.Parse()
 
 	viper.SetDefault("bind", ":8080")
+	viper.SetDefault("cacheSize", 1024)
+	viper.SetDefault("topChoices", 3)
 
 	viper.SetConfigName("dlrouter")        // name of config file (without extension)
 	viper.SetConfigType("yaml")            // REQUIRED if the config file does not have the extension in the name
@@ -64,45 +98,7 @@ func main() {
 		viper.SetConfigFile(*configFlag)
 	}
 
-	err := viper.ReadInConfig() // Find and read the config file
-
-	if err != nil { // Handle errors reading the config file
-		log.WithError(err).Fatalln("Unable to load config file")
-	}
-
-	db, err = maxminddb.Open(viper.GetString("geodb"))
-
-	if err != nil {
-		log.WithError(err).Fatalln("Unable to open database")
-	}
-
-	if mapFile := viper.GetString("dl_map"); mapFile != "" {
-		log.WithField("file", mapFile).Info("Loading download map")
-
-		dlMap, err = loadMap(mapFile)
-
-		if err != nil {
-			log.WithError(err).Fatalln("Unable to load download map")
-		}
-	}
-
-	serverList := viper.GetStringSlice("servers")
-
-	var wg sync.WaitGroup
-
-	for _, server := range serverList {
-		wg.Add(1)
-
-		go func(server string) {
-			defer wg.Done()
-
-			addServer(server)
-		}(server)
-	}
-
-	wg.Wait()
-
-	log.Info("Servers added, checking statuses")
+	reloadConfig()
 
 	// Start check loop
 	go servers.checkLoop()
@@ -114,15 +110,22 @@ func main() {
 	r.Use(RealIPMiddleware)
 	r.Use(logger.Logger("router", log.StandardLogger()))
 
+	r.Head("/status", statusHandler)
 	r.Get("/status", statusHandler)
-	r.Get("/mirrors", mirrorsHandler)
+	r.Get("/mirrors", legacyMirrorsHandler)
+	r.Get("/mirrors/{server}.svg", mirrorStatusHandler)
+	r.Get("/mirrors.json", mirrorsHandler)
 	r.Post("/reload", reloadHandler)
 	r.Get("/dl_map", dlMapHandler)
+	r.Get("/geoip", geoIPHandler)
 	r.Get("/metrics", promhttp.Handler().ServeHTTP)
-	r.HandleFunc("/*", redirectHandler)
+
+	r.NotFound(redirectHandler)
 
 	go http.ListenAndServe(viper.GetString("bind"), r)
 
+	log.Info("Ready")
+
 	c := make(chan os.Signal)
 
 	signal.Notify(c, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGHUP)
@@ -134,85 +137,6 @@ func main() {
 			break
 		}
 
-		reloadMap()
+		reloadConfig()
 	}
 }
-
-var metricReplacer = strings.NewReplacer(".", "_", "-", "_")
-
-func addServer(server string) {
-	var prefix string
-
-	if !strings.HasPrefix(server, "http") {
-		prefix = "https://"
-	}
-
-	u, err := url.Parse(prefix + server)
-
-	if err != nil {
-		log.WithFields(log.Fields{
-			"error":  err,
-			"server": server,
-		}).Warning("Server is invalid")
-		return
-	}
-
-	ips, err := net.LookupIP(u.Host)
-
-	if err != nil {
-		log.WithFields(log.Fields{
-			"error":  err,
-			"server": server,
-		}).Warning("Could not resolve address")
-		return
-	}
-
-	var city City
-	err = db.Lookup(ips[0], &city)
-
-	if err != nil {
-		log.WithFields(log.Fields{
-			"error":  err,
-			"server": server,
-		}).Warning("Could not geolocate address")
-		return
-	}
-
-	redirects := promauto.NewCounter(prometheus.CounterOpts{
-		Name: "armbian_router_redirects_" + metricReplacer.Replace(u.Host),
-		Help: "The number of redirects for server " + u.Host,
-	})
-
-	servers = append(servers, &Server{
-		Host:      u.Host,
-		Path:      u.Path,
-		Latitude:  city.Location.Latitude,
-		Longitude: city.Location.Longitude,
-		Redirects: redirects,
-	})
-
-	log.WithFields(log.Fields{
-		"server":    u.Host,
-		"path":      u.Path,
-		"latitude":  city.Location.Latitude,
-		"longitude": city.Location.Longitude,
-	}).Info("Added server")
-}
-
-func reloadMap() {
-	mapFile := viper.GetString("dl_map")
-
-	if mapFile == "" {
-		return
-	}
-
-	log.WithField("file", mapFile).Info("Loading download map")
-
-	newMap, err := loadMap(mapFile)
-
-	if err != nil {
-		return
-	}
-
-	dlMap = newMap
-}

map.go

@@ -4,7 +4,6 @@ import (
 	"encoding/csv"
 	"io"
 	"os"
-	"strings"
 )
 
 func loadMap(file string) (map[string]string, error) {
@@ -33,7 +32,7 @@ func loadMap(file string) (map[string]string, error) {
 			return nil, err
 		}
 
-		m[strings.TrimLeft(row[0], "/")] = strings.TrimLeft(row[1], "/")
+		m[row[0]] = row[1]
 	}
 
 	return m, nil

middleware.go

@@ -29,7 +29,9 @@ func RealIPMiddleware(f http.Handler) http.Handler {
 			return
 		}
 
-		if !net.ParseIP(host).IsPrivate() {
+		netIP := net.ParseIP(host)
+
+		if !netIP.IsLoopback() && !netIP.IsPrivate() {
 			f.ServeHTTP(w, r)
 			return
 		}

mirrors.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	_ "embed"
+	"encoding/json"
+	"github.com/go-chi/chi/v5"
+	"net/http"
+	"strings"
+)
+
+func legacyMirrorsHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	mirrorOutput := make(map[string][]string)
+
+	for region, mirrors := range regionMap {
+		list := make([]string, len(mirrors))
+
+		for i, mirror := range mirrors {
+			list[i] = r.URL.Scheme + "://" + mirror.Host + "/" + strings.TrimLeft(mirror.Path, "/")
+		}
+
+		mirrorOutput[region] = list
+	}
+
+	json.NewEncoder(w).Encode(mirrorOutput)
+}
+
+func mirrorsHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(servers)
+}
+
+var (
+	//go:embed assets/status-up.svg
+	statusUp []byte
+
+	//go:embed assets/status-down.svg
+	statusDown []byte
+
+	//go:embed assets/status-unknown.svg
+	statusUnknown []byte
+)
+
+func mirrorStatusHandler(w http.ResponseWriter, r *http.Request) {
+	serverHost := chi.URLParam(r, "server")
+
+	w.Header().Set("Content-Type", "image/svg+xml;charset=utf-8")
+
+	if serverHost == "" {
+		w.Write(statusUnknown)
+		return
+	}
+
+	serverHost = strings.Replace(serverHost, "_", ".", -1)
+
+	server, ok := hostMap[serverHost]
+
+	if !ok {
+		w.Write(statusUnknown)
+		return
+	}
+
+	if server.Available {
+		w.Write(statusUp)
+	} else {
+		w.Write(statusDown)
+	}
+}

servers.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"github.com/jmcvetta/randutil"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"math"
 	"net"
 	"net/http"
 	"runtime"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -14,17 +16,20 @@ import (
 
 var (
 	checkClient = &http.Client{
-		Timeout: 10 * time.Second,
+		Timeout: 20 * time.Second,
 	}
 )
 
 type Server struct {
-	Available bool
-	Host      string
-	Path      string
-	Latitude  float64
-	Longitude float64
-	Redirects prometheus.Counter
+	Available  bool               `json:"available"`
+	Host       string             `json:"host"`
+	Path       string             `json:"path"`
+	Latitude   float64            `json:"latitude"`
+	Longitude  float64            `json:"longitude"`
+	Weight     int                `json:"weight"`
+	Continent  string             `json:"continent"`
+	Redirects  prometheus.Counter `json:"-"`
+	LastChange time.Time          `json:"lastChange"`
 }
 
 func (server *Server) checkStatus() {
@@ -51,6 +56,7 @@ func (server *Server) checkStatus() {
 			}).Info("Server went offline")
 
 			server.Available = false
+			server.LastChange = time.Now()
 		} else {
 			log.WithFields(log.Fields{
 				"server": server.Host,
@@ -68,6 +74,7 @@ func (server *Server) checkStatus() {
 	if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound || res.StatusCode == http.StatusNotFound {
 		if !server.Available {
 			server.Available = true
+			server.LastChange = time.Now()
 			log.WithFields(responseFields).Info("Server is online")
 		}
 	} else {
@@ -76,6 +83,7 @@ func (server *Server) checkStatus() {
 		if server.Available {
 			log.WithFields(responseFields).Info("Server went offline")
 			server.Available = false
+			server.LastChange = time.Now()
 		}
 	}
 }
@@ -86,8 +94,8 @@ func (s ServerList) checkLoop() {
 	t := time.NewTicker(60 * time.Second)
 
 	for {
-		s.Check()
 		<-t.C
+		s.Check()
 	}
 }
 
@@ -110,33 +118,76 @@ func (s ServerList) Check() {
 	wg.Wait()
 }
 
-// Closest will use GeoIP on the IP provided and find the closest server.
+// ComputedDistance is a wrapper that contains a Server and Distance.
+type ComputedDistance struct {
+	Server   *Server
+	Distance float64
+}
+
+// DistanceList is a list of Computed Distances with an easy "Choices" func
+type DistanceList []ComputedDistance
+
+// Closest will use GeoIP on the IP provided and find the closest servers.
+// When we have a list of x servers closest, we can choose a random or weighted one.
 // Return values are the closest server, the distance, and if an error occurred.
 func (s ServerList) Closest(ip net.IP) (*Server, float64, error) {
-	var city City
-	err := db.Lookup(ip, &city)
+	choiceInterface, exists := serverCache.Get(ip.String())
+
+	if !exists {
+		var city LocationLookup
+		err := db.Lookup(ip, &city)
+
+		if err != nil {
+			return nil, -1, err
+		}
+
+		c := make(DistanceList, len(s))
+
+		for i, server := range s {
+			if !server.Available {
+				continue
+			}
+
+			distance := Distance(city.Location.Latitude, city.Location.Longitude, server.Latitude, server.Longitude)
+
+			c[i] = ComputedDistance{
+				Server:   server,
+				Distance: distance,
+			}
+		}
+
+		// Sort by distance
+		sort.Slice(c, func(i int, j int) bool {
+			return c[i].Distance < c[j].Distance
+		})
+
+		choices := make([]randutil.Choice, topChoices)
+
+		for i, item := range c[0:topChoices] {
+			if item.Server == nil {
+				continue
+			}
+
+			choices[i] = randutil.Choice{
+				Weight: item.Server.Weight,
+				Item:   item,
+			}
+		}
+
+		choiceInterface = choices
+
+		serverCache.Add(ip.String(), choiceInterface)
+	}
+
+	choice, err := randutil.WeightedChoice(choiceInterface.([]randutil.Choice))
 
 	if err != nil {
 		return nil, -1, err
 	}
 
-	var closest *Server
-	var closestDistance float64 = -1
+	dist := choice.Item.(ComputedDistance)
 
-	for _, server := range s {
-		if !server.Available {
-			continue
-		}
-
-		distance := Distance(city.Location.Latitude, city.Location.Longitude, server.Latitude, server.Longitude)
-
-		if closestDistance == -1 || distance < closestDistance {
-			closestDistance = distance
-			closest = server
-		}
-	}
-
-	return closest, closestDistance, nil
+	return dist.Server, dist.Distance, nil
 }
 
 // haversin(θ) function