Mimic existing mirrors endpoint, add geoip

config.go

@@ -150,36 +150,42 @@ func addServer(server ServerConfig, u *url.URL) *Server {
 		Path:      u.Path,
 		Latitude:  server.Latitude,
 		Longitude: server.Longitude,
+		Continent: server.Continent,
 		Weight:    server.Weight,
 	}
 
+	// Defaults to 10 to allow servers to be set lower for lower priority
 	if s.Weight == 0 {
-		s.Weight = 1
+		s.Weight = 10
+	}
+
+	ips, err := net.LookupIP(u.Host)
+
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":  err,
+			"server": s.Host,
+		}).Warning("Could not resolve address")
+		return nil
+	}
+
+	var city City
+	err = db.Lookup(ips[0], &city)
+
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":  err,
+			"server": s.Host,
+			"ip":     ips[0],
+		}).Warning("Could not geolocate address")
+		return nil
+	}
+
+	if s.Continent == "" {
+		s.Continent = city.Continent.Code
 	}
 
 	if s.Latitude == 0 && s.Longitude == 0 {
-		ips, err := net.LookupIP(u.Host)
-
-		if err != nil {
-			log.WithFields(log.Fields{
-				"error":  err,
-				"server": s.Host,
-			}).Warning("Could not resolve address")
-			return nil
-		}
-
-		var city City
-		err = db.Lookup(ips[0], &city)
-
-		if err != nil {
-			log.WithFields(log.Fields{
-				"error":  err,
-				"server": s.Host,
-				"ip":     ips[0],
-			}).Warning("Could not geolocate address")
-			return nil
-		}
-
 		s.Latitude = city.Location.Latitude
 		s.Longitude = city.Location.Longitude
 	}

dlrouter.yaml

@@ -1,5 +1,6 @@
 # GeoIP Database Path
 geodb: GeoLite2-City.mmdb
+dl_map: userdata.csv
 
 # LRU Cache Size (in items)
 cacheSize: 1024
@@ -13,25 +14,26 @@ cacheSize: 1024
 servers:
     - server: armbian.12z.eu/apt/
     - server: armbian.chi.auroradev.org/apt/
-      weight: 5
+      weight: 15
       latitude: 41.8879
       longitude: -88.1995
     - server: armbian.hosthatch.com/apt/
     - server: armbian.lv.auroradev.org/apt/
-      weight: 5
+      weight: 15
     - server: armbian.site-meganet.com/apt/
     - server: armbian.systemonachip.net/apt/
     - server: armbian.tnahosting.net/apt/
-      weight: 5
+      weight: 15
     - server: au-mirror.bret.dk/armbian/apt/
     - server: es-mirror.bret.dk/armbian/apt/
     - server: imola.armbian.com/apt/
     - server: mirror.iscas.ac.cn/armbian/
     - server: mirror.sjtu.edu.cn/armbian/
     - server: mirrors.aliyun.com/armbian/
+      continent: AS
     - server: mirrors.bfsu.edu.cn/armbian/
     - server: mirrors.dotsrc.org/armbian-apt/
-      weight: 5
+      weight: 15
     - server: mirrors.netix.net/armbian/apt/
     - server: mirrors.nju.edu.cn/armbian/
     - server: mirrors.sustech.edu.cn/armbian/
@@ -41,3 +43,5 @@ servers:
     - server: sg-mirror.bret.dk/armbian/apt/
     - server: stpete-mirror.armbian.com/apt/
     - server: xogium.performanceservers.nl/apt/
+    - server: github.com/armbian/mirror/releases/download/
+      continent: GITHUB

http.go

@@ -7,16 +7,35 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"strings"
 )
 
 func statusHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("OK"))
+}
+
+func legacyMirrorsHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	mirrors := make(map[string][]string)
+
+	for _, server := range servers {
+		u := &url.URL{
+			Scheme: r.URL.Scheme,
+			Host:   server.Host,
+			Path:   server.Path,
+		}
+
+		mirrors[server.Continent] = append(mirrors[server.Continent], u.String())
+	}
+
+	mirrors["default"] = append(mirrors["NA"], mirrors["EU"]...)
+
+	json.NewEncoder(w).Encode(mirrors)
 }
 
 func mirrorsHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
-
 	json.NewEncoder(w).Encode(servers)
 }
 
@@ -51,7 +70,13 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 	redirectPath := path.Join(server.Path, r.URL.Path)
 
 	if dlMap != nil {
-		if newPath, exists := dlMap[strings.TrimLeft(r.URL.Path, "/")]; exists {
+		p := r.URL.Path
+
+		if p[0] != '/' {
+			p = "/" + p
+		}
+
+		if newPath, exists := dlMap[p]; exists {
 			downloadsMapped.Inc()
 			redirectPath = path.Join(server.Path, newPath)
 		}
@@ -72,7 +97,7 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func reloadHandler(w http.ResponseWriter, r *http.Request) {
-	reloadMap()
+	reloadConfig()
 
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte("OK"))
@@ -88,3 +113,25 @@ func dlMapHandler(w http.ResponseWriter, r *http.Request) {
 
 	json.NewEncoder(w).Encode(dlMap)
 }
+
+func geoIPHandler(w http.ResponseWriter, r *http.Request) {
+	ipStr, _, err := net.SplitHostPort(r.RemoteAddr)
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	ip := net.ParseIP(ipStr)
+
+	var city City
+	err = db.Lookup(ip, &city)
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(city)
+}

main.go

@@ -38,18 +38,42 @@ var (
 	topChoices int
 )
 
-// City represents a MaxmindDB city
+type LocationLookup struct {
-type City struct {
 	Location struct {
 		Latitude  float64 `maxminddb:"latitude"`
 		Longitude float64 `maxminddb:"longitude"`
 	} `maxminddb:"location"`
 }
 
+// City represents a MaxmindDB city
+type City struct {
+	Continent struct {
+		Code      string            `maxminddb:"code" json:"code"`
+		GeoNameID uint              `maxminddb:"geoname_id" json:"geoname_id"`
+		Names     map[string]string `maxminddb:"names" json:"names"`
+	} `maxminddb:"continent" json:"continent"`
+	Country struct {
+		GeoNameID uint              `maxminddb:"geoname_id" json:"geoname_id"`
+		IsoCode   string            `maxminddb:"iso_code" json:"iso_code"`
+		Names     map[string]string `maxminddb:"names" json:"names"`
+	} `maxminddb:"country" json:"country"`
+	Location struct {
+		AccuracyRadius uint16  `maxminddb:"accuracy_radius" json:'accuracy_radius'`
+		Latitude       float64 `maxminddb:"latitude" json:"latitude"`
+		Longitude      float64 `maxminddb:"longitude" json:"longitude"`
+	} `maxminddb:"location"`
+	RegisteredCountry struct {
+		GeoNameID uint              `maxminddb:"geoname_id" json:"geoname_id"`
+		IsoCode   string            `maxminddb:"iso_code" json:"iso_code"`
+		Names     map[string]string `maxminddb:"names" json:"names"`
+	} `maxminddb:"registered_country" json:"registered_country"`
+}
+
 type ServerConfig struct {
 	Server    string  `mapstructure:"server" yaml:"server"`
 	Latitude  float64 `mapstructure:"latitude" yaml:"latitude"`
 	Longitude float64 `mapstructure:"longitude" yaml:"longitude"`
+	Continent string  `mapstructure:"continent"`
 	Weight    int     `mapstructure:"weight" yaml:"weight"`
 }
 
@@ -87,9 +111,11 @@ func main() {
 	r.Use(logger.Logger("router", log.StandardLogger()))
 
 	r.Get("/status", statusHandler)
-	r.Get("/mirrors", mirrorsHandler)
+	r.Get("/mirrors", legacyMirrorsHandler)
+	r.Get("/mirrors.json", mirrorsHandler)
 	r.Post("/reload", reloadHandler)
 	r.Get("/dl_map", dlMapHandler)
+	r.Get("/geoip", geoIPHandler)
 	r.Get("/metrics", promhttp.Handler().ServeHTTP)
 
 	r.NotFound(redirectHandler)

map.go

@@ -4,7 +4,6 @@ import (
 	"encoding/csv"
 	"io"
 	"os"
-	"strings"
 )
 
 func loadMap(file string) (map[string]string, error) {
@@ -33,7 +32,7 @@ func loadMap(file string) (map[string]string, error) {
 			return nil, err
 		}
 
-		m[strings.TrimLeft(row[0], "/")] = strings.TrimLeft(row[1], "/")
+		m[row[0]] = row[1]
 	}
 
 	return m, nil

servers.go

@@ -21,13 +21,14 @@ var (
 )
 
 type Server struct {
-	Available bool
+	Available bool               `json:"available"`
-	Host      string
+	Host      string             `json:"host"`
-	Path      string
+	Path      string             `json:"path"`
-	Latitude  float64
+	Latitude  float64            `json:"latitude"`
-	Longitude float64
+	Longitude float64            `json:"longitude"`
-	Weight    int
+	Weight    int                `json:"weight"`
-	Redirects prometheus.Counter
+	Continent string             `json:"continent"`
+	Redirects prometheus.Counter `json:"-"`
 }
 
 func (server *Server) checkStatus() {
@@ -139,38 +140,40 @@ func (d DistanceList) Choices() []randutil.Choice {
 // When we have a list of x servers closest, we can choose a random or weighted one.
 // Return values are the closest server, the distance, and if an error occurred.
 func (s ServerList) Closest(ip net.IP) (*Server, float64, error) {
-	i, exists := serverCache.Get(ip.String())
+	choiceInterface, exists := serverCache.Get(ip.String())
 
-	if exists {
+	if !exists {
-		return i.(ComputedDistance).Server, i.(ComputedDistance).Distance, nil
+		var city LocationLookup
-	}
+		err := db.Lookup(ip, &city)
 
-	var city City
+		if err != nil {
-	err := db.Lookup(ip, &city)
+			return nil, -1, err
-
-	if err != nil {
-		return nil, -1, err
-	}
-
-	c := make(DistanceList, len(s))
-
-	for i, server := range s {
-		if !server.Available {
-			continue
 		}
 
-		c[i] = ComputedDistance{
+		c := make(DistanceList, len(s))
-			Server:   server,
+
-			Distance: Distance(city.Location.Latitude, city.Location.Longitude, server.Latitude, server.Longitude),
+		for i, server := range s {
+			if !server.Available {
+				continue
+			}
+
+			c[i] = ComputedDistance{
+				Server:   server,
+				Distance: Distance(city.Location.Latitude, city.Location.Longitude, server.Latitude, server.Longitude),
+			}
 		}
+
+		// Sort by distance
+		sort.Slice(s, func(i int, j int) bool {
+			return c[i].Distance < c[j].Distance
+		})
+
+		choiceInterface = c[0:topChoices].Choices()
+
+		serverCache.Add(ip.String(), choiceInterface)
 	}
 
-	// Sort by distance
+	choice, err := randutil.WeightedChoice(choiceInterface.([]randutil.Choice))
-	sort.Slice(s, func(i int, j int) bool {
-		return c[i].Distance < c[j].Distance
-	})
-
-	choice, err := randutil.WeightedChoice(c[0:topChoices].Choices())
 
 	if err != nil {
 		return nil, -1, err
@@ -178,8 +181,6 @@ func (s ServerList) Closest(ip net.IP) (*Server, float64, error) {
 
 	dist := choice.Item.(ComputedDistance)
 
-	serverCache.Add(ip.String(), dist)
-
 	return dist.Server, dist.Distance, nil
 }