Resolve issue with returning a lower number of servers, add auth to reload

assets/status-down.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="82" height="20" role="img" aria-label="status: down"><title>status: down</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="82" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="43" height="20" fill="#555"/><rect x="43" width="39" height="20" fill="#e05d44"/><rect width="82" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="225" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="330">status</text><text x="225" y="140" transform="scale(.1)" fill="#fff" textLength="330">status</text><text aria-hidden="true" x="615" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="290">down</text><text x="615" y="140" transform="scale(.1)" fill="#fff" textLength="290">down</text></g></svg>

assets/status-unknown.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="104" height="20" role="img" aria-label="status: unknown"><title>status: unknown</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="104" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="43" height="20" fill="#555"/><rect x="43" width="61" height="20" fill="#9f9f9f"/><rect width="104" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="225" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="330">status</text><text x="225" y="140" transform="scale(.1)" fill="#fff" textLength="330">status</text><text aria-hidden="true" x="725" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="510">unknown</text><text x="725" y="140" transform="scale(.1)" fill="#fff" textLength="510">unknown</text></g></svg>

assets/status-up.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="66" height="20" role="img" aria-label="status: up"><title>status: up</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="66" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="43" height="20" fill="#555"/><rect x="43" width="23" height="20" fill="#4c1"/><rect width="66" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="225" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="330">status</text><text x="225" y="140" transform="scale(.1)" fill="#fff" textLength="330">status</text><text aria-hidden="true" x="535" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="130">up</text><text x="535" y="140" transform="scale(.1)" fill="#fff" textLength="130">up</text></g></svg>

config.go

@@ -14,6 +14,8 @@ import (
 )
 
 func reloadConfig() {
+	log.Info("Loading configuration...")
+
 	err := viper.ReadInConfig() // Find and read the config file
 
 	if err != nil { // Handle errors reading the config file
@@ -46,6 +48,25 @@ func reloadConfig() {
 	// Reload server list
 	reloadServers()
 
+	// Create mirror map
+	mirrors := make(map[string][]*Server)
+
+	for _, server := range servers {
+		mirrors[server.Continent] = append(mirrors[server.Continent], server)
+	}
+
+	mirrors["default"] = append(mirrors["NA"], mirrors["EU"]...)
+
+	regionMap = mirrors
+
+	hosts := make(map[string]*Server)
+
+	for _, server := range servers {
+		hosts[server.Host] = server
+	}
+
+	hostMap = hosts
+
 	// Check top choices size
 	if topChoices > len(servers) {
 		topChoices = len(servers)
@@ -150,14 +171,15 @@ func addServer(server ServerConfig, u *url.URL) *Server {
 		Path:      u.Path,
 		Latitude:  server.Latitude,
 		Longitude: server.Longitude,
+		Continent: server.Continent,
 		Weight:    server.Weight,
 	}
 
+	// Defaults to 10 to allow servers to be set lower for lower priority
 	if s.Weight == 0 {
-		s.Weight = 1
+		s.Weight = 10
 	}
 
-	if s.Latitude == 0 && s.Longitude == 0 {
 	ips, err := net.LookupIP(u.Host)
 
 	if err != nil {
@@ -180,6 +202,11 @@ func addServer(server ServerConfig, u *url.URL) *Server {
 		return nil
 	}
 
+	if s.Continent == "" {
+		s.Continent = city.Continent.Code
+	}
+
+	if s.Latitude == 0 && s.Longitude == 0 {
 		s.Latitude = city.Location.Latitude
 		s.Longitude = city.Location.Longitude
 	}

dlrouter.yaml

@@ -1,5 +1,6 @@
 # GeoIP Database Path
 geodb: GeoLite2-City.mmdb
+dl_map: userdata.csv
 
 # LRU Cache Size (in items)
 cacheSize: 1024
@@ -13,25 +14,26 @@ cacheSize: 1024
 servers:
     - server: armbian.12z.eu/apt/
     - server: armbian.chi.auroradev.org/apt/
-      weight: 5
+      weight: 15
       latitude: 41.8879
       longitude: -88.1995
     - server: armbian.hosthatch.com/apt/
     - server: armbian.lv.auroradev.org/apt/
-      weight: 5
+      weight: 15
     - server: armbian.site-meganet.com/apt/
     - server: armbian.systemonachip.net/apt/
     - server: armbian.tnahosting.net/apt/
-      weight: 5
+      weight: 15
     - server: au-mirror.bret.dk/armbian/apt/
     - server: es-mirror.bret.dk/armbian/apt/
     - server: imola.armbian.com/apt/
     - server: mirror.iscas.ac.cn/armbian/
     - server: mirror.sjtu.edu.cn/armbian/
     - server: mirrors.aliyun.com/armbian/
+      continent: AS
     - server: mirrors.bfsu.edu.cn/armbian/
     - server: mirrors.dotsrc.org/armbian-apt/
-      weight: 5
+      weight: 15
     - server: mirrors.netix.net/armbian/apt/
     - server: mirrors.nju.edu.cn/armbian/
     - server: mirrors.sustech.edu.cn/armbian/
@@ -41,3 +43,5 @@ servers:
     - server: sg-mirror.bret.dk/armbian/apt/
     - server: stpete-mirror.armbian.com/apt/
     - server: xogium.performanceservers.nl/apt/
+    - server: github.com/armbian/mirror/releases/download/
+      continent: GITHUB

http.go

@@ -3,21 +3,19 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/jmcvetta/randutil"
+	"github.com/spf13/viper"
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strings"
 )
 
 func statusHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
-}
-
-func mirrorsHandler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json")
-
-	json.NewEncoder(w).Encode(servers)
+	w.Write([]byte("OK"))
 }
 
 func redirectHandler(w http.ResponseWriter, r *http.Request) {
@@ -30,18 +28,59 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 
 	ip := net.ParseIP(ipStr)
 
-	// TODO: This is temporary to allow testing on private addresses.
-	if ip.IsPrivate() {
-		ip = net.ParseIP("1.1.1.1")
+	if ip.IsLoopback() || ip.IsPrivate() {
+		overrideIP := os.Getenv("OVERRIDE_IP")
+
+		if overrideIP == "" {
+			overrideIP = "1.1.1.1"
 		}
 
-	server, distance, err := servers.Closest(ip)
+		ip = net.ParseIP(overrideIP)
+	}
+
+	var server *Server
+	var distance float64
+
+	if strings.HasPrefix(r.URL.Path, "/region") {
+		parts := strings.Split(r.URL.Path, "/")
+
+		// region = parts[2]
+		if mirrors, ok := regionMap[parts[2]]; ok {
+			choices := make([]randutil.Choice, len(mirrors))
+
+			for i, item := range mirrors {
+				if !item.Available {
+					continue
+				}
+
+				choices[i] = randutil.Choice{
+					Weight: item.Weight,
+					Item:   item,
+				}
+			}
+
+			choice, err := randutil.WeightedChoice(choices)
 
 			if err != nil {
 				http.Error(w, err.Error(), http.StatusInternalServerError)
 				return
 			}
 
+			server = choice.Item.(*Server)
+
+			r.URL.Path = strings.Join(parts[3:], "/")
+		}
+	}
+
+	if server == nil {
+		server, distance, err = servers.Closest(ip)
+
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+
 	scheme := r.URL.Scheme
 
 	if scheme == "" {
@@ -57,6 +96,10 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if strings.HasSuffix(r.URL.Path, "/") && !strings.HasSuffix(redirectPath, "/") {
+		redirectPath += "/"
+	}
+
 	u := &url.URL{
 		Scheme: scheme,
 		Host:   server.Host,
@@ -66,13 +109,30 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 	server.Redirects.Inc()
 	redirectsServed.Inc()
 
+	if distance > 0 {
 		w.Header().Set("X-Geo-Distance", fmt.Sprintf("%f", distance))
+	}
+
 	w.Header().Set("Location", u.String())
 	w.WriteHeader(http.StatusFound)
 }
 
 func reloadHandler(w http.ResponseWriter, r *http.Request) {
-	reloadMap()
+	token := r.Header.Get("Authorization")
+
+	if token == "" || !strings.HasPrefix(token, "Bearer") || !strings.Contains(token, " ") {
+		w.WriteHeader(http.StatusUnauthorized)
+		return
+	}
+
+	token = token[strings.Index(token, " ")+1:]
+
+	if token != viper.GetString("reloadToken") {
+		w.WriteHeader(http.StatusUnauthorized)
+		return
+	}
+
+	reloadConfig()
 
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte("OK"))
@@ -88,3 +148,25 @@ func dlMapHandler(w http.ResponseWriter, r *http.Request) {
 
 	json.NewEncoder(w).Encode(dlMap)
 }
+
+func geoIPHandler(w http.ResponseWriter, r *http.Request) {
+	ipStr, _, err := net.SplitHostPort(r.RemoteAddr)
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	ip := net.ParseIP(ipStr)
+
+	var city City
+	err = db.Lookup(ip, &city)
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(city)
+}

main.go

@@ -20,8 +20,10 @@ import (
 var (
 	db         *maxminddb.Reader
 	servers    ServerList
-
+	regionMap  map[string][]*Server
+	hostMap    map[string]*Server
 	dlMap      map[string]string
+	topChoices int
 
 	redirectsServed = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "armbian_router_redirects",
@@ -34,35 +36,63 @@ var (
 	})
 
 	serverCache *lru.Cache
-
-	topChoices int
 )
 
-// City represents a MaxmindDB city
-type City struct {
+type LocationLookup struct {
 	Location struct {
 		Latitude  float64 `maxminddb:"latitude"`
 		Longitude float64 `maxminddb:"longitude"`
 	} `maxminddb:"location"`
 }
 
+// City represents a MaxmindDB city
+type City struct {
+	Continent struct {
+		Code      string            `maxminddb:"code" json:"code"`
+		GeoNameID uint              `maxminddb:"geoname_id" json:"geoname_id"`
+		Names     map[string]string `maxminddb:"names" json:"names"`
+	} `maxminddb:"continent" json:"continent"`
+	Country struct {
+		GeoNameID uint              `maxminddb:"geoname_id" json:"geoname_id"`
+		IsoCode   string            `maxminddb:"iso_code" json:"iso_code"`
+		Names     map[string]string `maxminddb:"names" json:"names"`
+	} `maxminddb:"country" json:"country"`
+	Location struct {
+		AccuracyRadius uint16  `maxminddb:"accuracy_radius" json:'accuracy_radius'`
+		Latitude       float64 `maxminddb:"latitude" json:"latitude"`
+		Longitude      float64 `maxminddb:"longitude" json:"longitude"`
+	} `maxminddb:"location"`
+	RegisteredCountry struct {
+		GeoNameID uint              `maxminddb:"geoname_id" json:"geoname_id"`
+		IsoCode   string            `maxminddb:"iso_code" json:"iso_code"`
+		Names     map[string]string `maxminddb:"names" json:"names"`
+	} `maxminddb:"registered_country" json:"registered_country"`
+}
+
 type ServerConfig struct {
 	Server    string  `mapstructure:"server" yaml:"server"`
 	Latitude  float64 `mapstructure:"latitude" yaml:"latitude"`
 	Longitude float64 `mapstructure:"longitude" yaml:"longitude"`
+	Continent string  `mapstructure:"continent"`
 	Weight    int     `mapstructure:"weight" yaml:"weight"`
 }
 
 var (
 	configFlag = flag.String("config", "", "configuration file path")
+	flagDebug  = flag.Bool("debug", false, "Enable debug logging")
 )
 
 func main() {
 	flag.Parse()
 
+	if *flagDebug {
+		log.SetLevel(log.DebugLevel)
+	}
+
 	viper.SetDefault("bind", ":8080")
 	viper.SetDefault("cacheSize", 1024)
 	viper.SetDefault("topChoices", 3)
+	viper.SetDefault("reloadKey", randSeq(32))
 
 	viper.SetConfigName("dlrouter")        // name of config file (without extension)
 	viper.SetConfigType("yaml")            // REQUIRED if the config file does not have the extension in the name
@@ -86,10 +116,14 @@ func main() {
 	r.Use(RealIPMiddleware)
 	r.Use(logger.Logger("router", log.StandardLogger()))
 
+	r.Head("/status", statusHandler)
 	r.Get("/status", statusHandler)
-	r.Get("/mirrors", mirrorsHandler)
+	r.Get("/mirrors", legacyMirrorsHandler)
+	r.Get("/mirrors/{server}.svg", mirrorStatusHandler)
+	r.Get("/mirrors.json", mirrorsHandler)
 	r.Post("/reload", reloadHandler)
 	r.Get("/dl_map", dlMapHandler)
+	r.Get("/geoip", geoIPHandler)
 	r.Get("/metrics", promhttp.Handler().ServeHTTP)
 
 	r.NotFound(redirectHandler)

mirrors.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	_ "embed"
+	"encoding/json"
+	"github.com/go-chi/chi/v5"
+	"net/http"
+	"strings"
+)
+
+func legacyMirrorsHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	mirrorOutput := make(map[string][]string)
+
+	for region, mirrors := range regionMap {
+		list := make([]string, len(mirrors))
+
+		for i, mirror := range mirrors {
+			list[i] = r.URL.Scheme + "://" + mirror.Host + "/" + strings.TrimLeft(mirror.Path, "/")
+		}
+
+		mirrorOutput[region] = list
+	}
+
+	json.NewEncoder(w).Encode(mirrorOutput)
+}
+
+func mirrorsHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(servers)
+}
+
+var (
+	//go:embed assets/status-up.svg
+	statusUp []byte
+
+	//go:embed assets/status-down.svg
+	statusDown []byte
+
+	//go:embed assets/status-unknown.svg
+	statusUnknown []byte
+)
+
+func mirrorStatusHandler(w http.ResponseWriter, r *http.Request) {
+	serverHost := chi.URLParam(r, "server")
+
+	w.Header().Set("Content-Type", "image/svg+xml;charset=utf-8")
+
+	if serverHost == "" {
+		w.Write(statusUnknown)
+		return
+	}
+
+	serverHost = strings.Replace(serverHost, "_", ".", -1)
+
+	server, ok := hostMap[serverHost]
+
+	if !ok {
+		w.Write(statusUnknown)
+		return
+	}
+
+	if server.Available {
+		w.Write(statusUp)
+	} else {
+		w.Write(statusDown)
+	}
+}

servers.go

@@ -7,6 +7,7 @@ import (
 	"math"
 	"net"
 	"net/http"
+	"net/url"
 	"runtime"
 	"sort"
 	"strings"
@@ -16,22 +17,24 @@ import (
 
 var (
 	checkClient = &http.Client{
-		Timeout: 10 * time.Second,
+		Timeout: 20 * time.Second,
 	}
 )
 
 type Server struct {
-	Available bool
-	Host      string
-	Path      string
-	Latitude  float64
-	Longitude float64
-	Weight    int
-	Redirects prometheus.Counter
+	Available  bool               `json:"available"`
+	Host       string             `json:"host"`
+	Path       string             `json:"path"`
+	Latitude   float64            `json:"latitude"`
+	Longitude  float64            `json:"longitude"`
+	Weight     int                `json:"weight"`
+	Continent  string             `json:"continent"`
+	Redirects  prometheus.Counter `json:"-"`
+	LastChange time.Time          `json:"lastChange"`
 }
 
 func (server *Server) checkStatus() {
-	req, err := http.NewRequest(http.MethodGet, "https://"+server.Host+"/"+strings.TrimLeft(server.Path, "/"), nil)
+	req, err := http.NewRequest(http.MethodGet, "http://"+server.Host+"/"+strings.TrimLeft(server.Path, "/"), nil)
 
 	req.Header.Set("User-Agent", "ArmbianRouter/1.0 (Go "+runtime.Version()+")")
 
@@ -54,6 +57,7 @@ func (server *Server) checkStatus() {
 			}).Info("Server went offline")
 
 			server.Available = false
+			server.LastChange = time.Now()
 		} else {
 			log.WithFields(log.Fields{
 				"server": server.Host,
@@ -69,8 +73,38 @@ func (server *Server) checkStatus() {
 	}
 
 	if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound || res.StatusCode == http.StatusNotFound {
+		if res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound {
+			location := res.Header.Get("Location")
+
+			responseFields["url"] = location
+
+			log.WithFields(responseFields).Debug("Server responded with redirect")
+
+			newUrl, err := url.Parse(location)
+
+			if err != nil {
+				if server.Available {
+					log.WithFields(responseFields).Warning("Server returned invalid url")
+					server.Available = false
+					server.LastChange = time.Now()
+				}
+				return
+			}
+
+			if newUrl.Scheme == "https" {
+				if server.Available {
+					responseFields["url"] = location
+					log.WithFields(responseFields).Warning("Server returned https url for http request")
+					server.Available = false
+					server.LastChange = time.Now()
+				}
+				return
+			}
+		}
+
 		if !server.Available {
 			server.Available = true
+			server.LastChange = time.Now()
 			log.WithFields(responseFields).Info("Server is online")
 		}
 	} else {
@@ -79,6 +113,7 @@ func (server *Server) checkStatus() {
 		if server.Available {
 			log.WithFields(responseFields).Info("Server went offline")
 			server.Available = false
+			server.LastChange = time.Now()
 		}
 	}
 }
@@ -122,30 +157,14 @@ type ComputedDistance struct {
 // DistanceList is a list of Computed Distances with an easy "Choices" func
 type DistanceList []ComputedDistance
 
-func (d DistanceList) Choices() []randutil.Choice {
-	c := make([]randutil.Choice, len(d))
-
-	for i, item := range d {
-		c[i] = randutil.Choice{
-			Weight: item.Server.Weight,
-			Item:   item,
-		}
-	}
-
-	return c
-}
-
 // Closest will use GeoIP on the IP provided and find the closest servers.
 // When we have a list of x servers closest, we can choose a random or weighted one.
 // Return values are the closest server, the distance, and if an error occurred.
 func (s ServerList) Closest(ip net.IP) (*Server, float64, error) {
-	i, exists := serverCache.Get(ip.String())
+	choiceInterface, exists := serverCache.Get(ip.String())
 
-	if exists {
-		return i.(ComputedDistance).Server, i.(ComputedDistance).Distance, nil
-	}
-
-	var city City
+	if !exists {
+		var city LocationLookup
 		err := db.Lookup(ip, &city)
 
 		if err != nil {
@@ -159,18 +178,44 @@ func (s ServerList) Closest(ip net.IP) (*Server, float64, error) {
 				continue
 			}
 
+			distance := Distance(city.Location.Latitude, city.Location.Longitude, server.Latitude, server.Longitude)
+
 			c[i] = ComputedDistance{
 				Server:   server,
-			Distance: Distance(city.Location.Latitude, city.Location.Longitude, server.Latitude, server.Longitude),
+				Distance: distance,
 			}
 		}
 
 		// Sort by distance
-	sort.Slice(s, func(i int, j int) bool {
+		sort.Slice(c, func(i int, j int) bool {
 			return c[i].Distance < c[j].Distance
 		})
 
-	choice, err := randutil.WeightedChoice(c[0:topChoices].Choices())
+		choiceCount := topChoices
+
+		if len(c) < topChoices {
+			choiceCount = len(c)
+		}
+
+		choices := make([]randutil.Choice, choiceCount)
+
+		for i, item := range c[0:choiceCount] {
+			if item.Server == nil {
+				continue
+			}
+
+			choices[i] = randutil.Choice{
+				Weight: item.Server.Weight,
+				Item:   item,
+			}
+		}
+
+		choiceInterface = choices
+
+		serverCache.Add(ip.String(), choiceInterface)
+	}
+
+	choice, err := randutil.WeightedChoice(choiceInterface.([]randutil.Choice))
 
 	if err != nil {
 		return nil, -1, err
@@ -178,8 +223,6 @@ func (s ServerList) Closest(ip net.IP) (*Server, float64, error) {
 
 	dist := choice.Item.(ComputedDistance)
 
-	serverCache.Add(ip.String(), dist)
-
 	return dist.Server, dist.Distance, nil
 }
 

util.go

@@ -0,0 +1,13 @@
+package main
+
+import "math/rand"
+
+var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+func randSeq(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}