Merge pull request 'Initial testing and improvements of code' (#1) from feature/testing into master

Reviewed-on: #1

.drone.yml

@@ -10,7 +10,11 @@ steps:
         path: /build
     commands:
       - go mod download
-      - ginkgo .
+      - go install github.com/onsi/ginkgo/v2/ginkgo
+      - ginkgo --randomize-all --p --cover --coverprofile=cover.out .
+      - go tool cover -func=cover.out
+    environment:
+      CGO_ENABLED: '0'
   - name: build
     image: tystuyfzand/goc:latest
     volumes:

README.md

@@ -15,4 +15,104 @@ Code Quality
 
 The code quality isn't the greatest/top tier. All code lives in the "main" package and should be moved at some point.
 
-Regardless, it is meant to be simple and easy to understand.
+Regardless, it is meant to be simple and easy to understand.
+
+Configuration
+-------------
+
+### Modes
+
+#### Redirect
+
+Standard redirect functionality
+
+#### Download Mapping
+
+Uses the `dl_map` configuration variable to enable mapping of paths to new paths.
+
+Think symlinks, but in a generated file.
+
+### Mirrors
+Mirror targets with trailing slash are placed in the yaml configuration file.
+
+### Example YAML
+```yaml
+# GeoIP Database Path
+geodb: GeoLite2-City.mmdb
+
+# Comment out to disable
+dl_map: userdata.csv
+
+# LRU Cache Size (in items)
+cacheSize: 1024
+
+# Server definition
+# Weights are just like nginx, where if it's > 1 it'll be chosen x out of x + total times
+# By default, the top 3 servers are used for choosing the best.
+# server = full url or host+path
+# weight = int
+# optional: latitude, longitude (float)
+servers:
+  - server: armbian.12z.eu/apt/
+  - server: armbian.chi.auroradev.org/apt/
+    weight: 15
+    latitude: 41.8879
+    longitude: -88.1995
+````
+
+## API
+
+`/status`
+
+Meant for a simple health check (nginx/etc can 502 or similar if down)
+
+`/reload`
+
+Flushes cache and reloads configuration and mapping. Requires reloadToken to be set in the configuration, and a matching token provided in `Authorization: Bearer TOKEN`
+
+`/mirrors`
+
+Shows all mirrors in the legacy (by region) format
+
+`/mirrors.json`
+
+Shows all mirrors in the new JSON format. Example:
+
+```json
+[
+  {
+    "available":true,
+    "host":"imola.armbian.com",
+    "path":"/apt/",
+    "latitude":46.0503,
+    "longitude":14.5046,
+    "weight":10,
+    "continent":"EU",
+    "lastChange":"2022-08-12T06:52:35.029565986Z"
+  }
+]
+```
+
+`/mirrors/{server}.svg`
+
+Magic SVG path to show badges based on server status, for use in dynamic mirror lists.
+
+`/dl_map`
+
+Shows json-encoded download mappings
+
+`/geoip`
+
+Shows GeoIP information for the requester
+
+`/region/REGIONCODE/PATH`
+
+Using this magic path will redirect to the desired region:
+
+* NA - North America
+* EU - Europe
+* AS - Asia
+
+`/metrics`
+
+Prometheus metrics endpoint. Metrics aren't considered private, thus are exposed to the public.

check.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	log "github.com/sirupsen/logrus"
+	"net"
 	"net/http"
 	"net/url"
 	"runtime"
@@ -82,7 +83,13 @@ func checkRedirect(locationHeader string) (bool, error) {
 
 // checkTLS checks tls certificates from a host, ensures they're valid, and not expired.
 func checkTLS(server *Server, logFields log.Fields) (bool, error) {
-	conn, err := tls.Dial("tcp", server.Host+":443", nil)
+	host, port, err := net.SplitHostPort(server.Host)
+
+	if port == "" {
+		port = "443"
+	}
+
+	conn, err := tls.Dial("tcp", host+":"+port, checkTLSConfig)
 
 	if err != nil {
 		return false, err

check_test.go

@@ -1,40 +1,88 @@
 package main
 
 import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	log "github.com/sirupsen/logrus"
+	"math/big"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"strings"
+	"time"
 )
 
+func genTestCerts(notBefore, notAfter time.Time) (*pem.Block, *pem.Block, error) {
+	// Create a Certificate Authority Cert
+	template := x509.Certificate{
+		SerialNumber:          big.NewInt(0),
+		Subject:               pkix.Name{CommonName: "localhost"},
+		SignatureAlgorithm:    x509.SHA256WithRSA,
+		NotBefore:             notBefore,
+		NotAfter:              notAfter,
+		BasicConstraintsValid: true,
+		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1")},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyAgreement | x509.KeyUsageKeyEncipherment | x509.KeyUsageDataEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+	}
+
+	// Create a Private Key
+	key, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Could not generate rsa key - %s", err)
+	}
+
+	// Use CA Cert to sign a CSR and create a Public Cert
+	csr := &key.PublicKey
+	cert, err := x509.CreateCertificate(rand.Reader, &template, &template, csr, key)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Could not generate certificate - %s", err)
+	}
+
+	// Convert keys into pem.Block
+	c := &pem.Block{Type: "CERTIFICATE", Bytes: cert}
+	k := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}
+	return c, k, nil
+}
+
 var _ = Describe("Check suite", func() {
+	var (
+		httpServer *httptest.Server
+		server     *Server
+		handler    http.HandlerFunc
+	)
+	BeforeEach(func() {
+		httpServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			handler(w, r)
+		}))
+	})
+	AfterEach(func() {
+		httpServer.Close()
+	})
+	setupServer := func() {
+		u, err := url.Parse(httpServer.URL)
+
+		if err != nil {
+			panic(err)
+		}
+		server = &Server{
+			Host: u.Host,
+			Path: u.Path,
+		}
+	}
+
 	Context("HTTP Checks", func() {
-		var (
-			httpServer *httptest.Server
-			server     *Server
-			handler    http.HandlerFunc
-		)
 		BeforeEach(func() {
-			httpServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				handler(w, r)
-			}))
-
-			u, err := url.Parse(httpServer.URL)
-
-			if err != nil {
-				panic(err)
-			}
-
-			server = &Server{
-				Host: u.Host,
-				Path: u.Path,
-			}
-		})
-		AfterEach(func() {
-			httpServer.Close()
+			httpServer.Start()
+			setupServer()
 		})
 		It("Should successfully check for connectivity", func() {
 			handler = func(w http.ResponseWriter, r *http.Request) {
@@ -59,6 +107,96 @@ var _ = Describe("Check suite", func() {
 		})
 	})
 	Context("TLS Checks", func() {
+		var (
+			x509Cert *x509.Certificate
+		)
+		setupCerts := func(notBefore, notAfter time.Time) {
+			cert, key, err := genTestCerts(notBefore, notAfter)
 
+			if err != nil {
+				panic("Unable to generate test certs")
+			}
+
+			x509Cert, err = x509.ParseCertificate(cert.Bytes)
+
+			if err != nil {
+				panic("Unable to parse certificate from bytes: " + err.Error())
+			}
+
+			tlsPair, err := tls.X509KeyPair(pem.EncodeToMemory(cert), pem.EncodeToMemory(key))
+
+			if err != nil {
+				panic("Unable to load tls key pair: " + err.Error())
+			}
+
+			httpServer.TLS = &tls.Config{
+				Certificates: []tls.Certificate{tlsPair},
+			}
+
+			httpServer.StartTLS()
+			setupServer()
+		}
+		Context("CA Tests", func() {
+			BeforeEach(func() {
+				setupCerts(time.Now(), time.Now().Add(24*time.Hour))
+			})
+			It("Should fail due to invalid ca", func() {
+				res, err := checkTLS(server, log.Fields{})
+
+				Expect(res).To(BeFalse())
+				Expect(err).ToNot(BeNil())
+			})
+			It("Should successfully validate certificates (valid ca, valid date/times, etc)", func() {
+				pool := x509.NewCertPool()
+
+				pool.AddCert(x509Cert)
+
+				checkTLSConfig = &tls.Config{RootCAs: pool}
+
+				res, err := checkTLS(server, log.Fields{})
+
+				Expect(res).To(BeFalse())
+				Expect(err).ToNot(BeNil())
+
+				checkTLSConfig = nil
+			})
+		})
+		Context("Expiration tests", func() {
+			AfterEach(func() {
+				checkTLSConfig = nil
+			})
+			It("Should fail due to not yet valid certificate", func() {
+				setupCerts(time.Now().Add(5*time.Hour), time.Now().Add(10*time.Hour))
+
+				// Trust our certs
+				pool := x509.NewCertPool()
+
+				pool.AddCert(x509Cert)
+
+				checkTLSConfig = &tls.Config{RootCAs: pool}
+
+				// Check TLS
+				res, err := checkTLS(server, log.Fields{})
+
+				Expect(res).To(BeFalse())
+				Expect(err).ToNot(BeNil())
+			})
+			It("Should fail due to expired certificate", func() {
+				setupCerts(time.Now().Add(-10*time.Hour), time.Now().Add(-5*time.Hour))
+
+				// Trust our certs
+				pool := x509.NewCertPool()
+
+				pool.AddCert(x509Cert)
+
+				checkTLSConfig = &tls.Config{RootCAs: pool}
+
+				// Check TLS
+				res, err := checkTLS(server, log.Fields{})
+
+				Expect(res).To(BeFalse())
+				Expect(err).ToNot(BeNil())
+			})
+		})
 	})
 })

config.go

@@ -3,6 +3,7 @@ package main
 import (
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/oschwald/maxminddb-golang"
+	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	log "github.com/sirupsen/logrus"
@@ -13,13 +14,13 @@ import (
 	"sync"
 )
 
-func reloadConfig() {
+func reloadConfig() error {
 	log.Info("Loading configuration...")
 
 	err := viper.ReadInConfig() // Find and read the config file
 
 	if err != nil { // Handle errors reading the config file
-		log.WithError(err).Fatalln("Unable to load config file")
+		return errors.Wrap(err, "Unable to read configuration")
 	}
 
 	// db will never be reloaded.
@@ -28,7 +29,7 @@ func reloadConfig() {
 		db, err = maxminddb.Open(viper.GetString("geodb"))
 
 		if err != nil {
-			log.WithError(err).Fatalln("Unable to open database")
+			return errors.Wrap(err, "Unable to open database")
 		}
 	}
 
@@ -46,10 +47,14 @@ func reloadConfig() {
 	topChoices = viper.GetInt("topChoices")
 
 	// Reload map file
-	reloadMap()
+	if err := reloadMap(); err != nil {
+		return errors.Wrap(err, "Unable to load map file")
+	}
 
 	// Reload server list
-	reloadServers()
+	if err := reloadServers(); err != nil {
+		return errors.Wrap(err, "Unable to load servers")
+	}
 
 	// Create mirror map
 	mirrors := make(map[string][]*Server)
@@ -77,11 +82,16 @@ func reloadConfig() {
 
 	// Force check
 	go servers.Check()
+
+	return nil
 }
 
-func reloadServers() {
+func reloadServers() error {
 	var serverList []ServerConfig
-	viper.UnmarshalKey("servers", &serverList)
+
+	if err := viper.UnmarshalKey("servers", &serverList); err != nil {
+		return err
+	}
 
 	var wg sync.WaitGroup
 
@@ -109,7 +119,7 @@ func reloadServers() {
 				"error":  err,
 				"server": server,
 			}).Warning("Server is invalid")
-			return
+			return err
 		}
 
 		hosts[u.Host] = true
@@ -161,6 +171,8 @@ func reloadServers() {
 
 		servers = append(servers[:i], servers[i+1:]...)
 	}
+
+	return nil
 }
 
 var metricReplacer = strings.NewReplacer(".", "_", "-", "_")
@@ -217,20 +229,22 @@ func addServer(server ServerConfig, u *url.URL) *Server {
 	return s
 }
 
-func reloadMap() {
+func reloadMap() error {
 	mapFile := viper.GetString("dl_map")
 
 	if mapFile == "" {
-		return
+		return nil
 	}
 
 	log.WithField("file", mapFile).Info("Loading download map")
 
-	newMap, err := loadMap(mapFile)
+	newMap, err := loadMapFile(mapFile)
 
 	if err != nil {
-		return
+		return err
 	}
 
 	dlMap = newMap
+
+	return nil
 }

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.1.4
 	github.com/onsi/gomega v1.20.0
 	github.com/oschwald/maxminddb-golang v1.8.0
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/viper v1.10.1

go.sum

@@ -128,6 +128,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -197,6 +198,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
@@ -322,6 +324,7 @@ github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhEC
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -677,6 +680,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
 golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

http.go

@@ -13,11 +13,18 @@ import (
 	"strings"
 )
 
+// statusHandler is a simple handler that will always return 200 OK with a body of "OK"
 func statusHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("OK"))
+
+	if r.Method != http.MethodHead {
+		w.Write([]byte("OK"))
+	}
 }
 
+// redirectHandler is the default "not found" handler which handles redirects
+// if the environment variable OVERRIDE_IP is set, it will use that ip address
+// this is useful for local testing when you're on the local network
 func redirectHandler(w http.ResponseWriter, r *http.Request) {
 	ipStr, _, err := net.SplitHostPort(r.RemoteAddr)
 
@@ -41,6 +48,8 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 	var server *Server
 	var distance float64
 
+	// If the path has a prefix of region/NA, it will use specific regions instead
+	// of the default geographical distance
 	if strings.HasPrefix(r.URL.Path, "/region") {
 		parts := strings.Split(r.URL.Path, "/")
 
@@ -72,6 +81,7 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	// If none of the above exceptions are matched, we use the geographical distance based on IP
 	if server == nil {
 		server, distance, err = servers.Closest(ip)
 
@@ -81,14 +91,19 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	// If we don't have a scheme, we'll use https by default
 	scheme := r.URL.Scheme
 
 	if scheme == "" {
 		scheme = "https"
 	}
 
+	// redirectPath is a combination of server path (which can be something like /armbian)
+	// and the URL path.
+	// Example: /armbian + /some/path = /armbian/some/path
 	redirectPath := path.Join(server.Path, r.URL.Path)
 
+	// If we have a dlMap, we map the url to a final path instead
 	if dlMap != nil {
 		if newPath, exists := dlMap[strings.TrimLeft(r.URL.Path, "/")]; exists {
 			downloadsMapped.Inc()
@@ -100,6 +115,7 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 		redirectPath += "/"
 	}
 
+	// We need to build the final url now
 	u := &url.URL{
 		Scheme: scheme,
 		Host:   server.Host,
@@ -109,6 +125,7 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 	server.Redirects.Inc()
 	redirectsServed.Inc()
 
+	// If we used geographical distance, we add an X-Geo-Distance header for debug.
 	if distance > 0 {
 		w.Header().Set("X-Geo-Distance", fmt.Sprintf("%f", distance))
 	}
@@ -117,7 +134,16 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusFound)
 }
 
+// reloadHandler is an http handler which lets us reload the server configuration
+// It is only enabled when the reloadToken is set in the configuration
 func reloadHandler(w http.ResponseWriter, r *http.Request) {
+	expectedToken := viper.GetString("reloadToken")
+
+	if expectedToken == "" {
+		w.WriteHeader(http.StatusUnauthorized)
+		return
+	}
+
 	token := r.Header.Get("Authorization")
 
 	if token == "" || !strings.HasPrefix(token, "Bearer") || !strings.Contains(token, " ") {
@@ -127,12 +153,16 @@ func reloadHandler(w http.ResponseWriter, r *http.Request) {
 
 	token = token[strings.Index(token, " ")+1:]
 
-	if token != viper.GetString("reloadToken") {
+	if token != expectedToken {
 		w.WriteHeader(http.StatusUnauthorized)
 		return
 	}
 
-	reloadConfig()
+	if err := reloadConfig(); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
 
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte("OK"))

main.go

@@ -104,7 +104,9 @@ func main() {
 		viper.SetConfigFile(*configFlag)
 	}
 
-	reloadConfig()
+	if err := reloadConfig(); err != nil {
+		log.WithError(err).Fatalln("Unable to load configuration")
+	}
 
 	// Start check loop
 	go servers.checkLoop()
@@ -143,6 +145,10 @@ func main() {
 			break
 		}
 
-		reloadConfig()
+		err := reloadConfig()
+
+		if err != nil {
+			log.WithError(err).Warning("Did not reload configuration due to error")
+		}
 	}
 }

map.go

@@ -7,7 +7,8 @@ import (
 	"strings"
 )
 
-func loadMap(file string) (map[string]string, error) {
+// loadMapFile loads a file as a map
+func loadMapFile(file string) (map[string]string, error) {
 	f, err := os.Open(file)
 
 	if err != nil {
@@ -16,6 +17,11 @@ func loadMap(file string) (map[string]string, error) {
 
 	defer f.Close()
 
+	return loadMap(f)
+}
+
+// loadMap loads a pipe separated file of mappings
+func loadMap(f io.Reader) (map[string]string, error) {
 	m := make(map[string]string)
 
 	r := csv.NewReader(f)

map_test.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"strings"
+)
+
+var _ = Describe("Map", func() {
+	It("Should successfully load the map", func() {
+		m, err := loadMap(strings.NewReader(`bananapi/Bullseye_current|bananapi/archive/Armbian_21.08.1_Bananapi_bullseye_current_5.10.60.img.xz|Aug 26 2021|332M`))
+
+		Expect(err).To(BeNil())
+		Expect(m["bananapi/Bullseye_current"]).To(Equal("bananapi/archive/Armbian_21.08.1_Bananapi_bullseye_current_5.10.60.img.xz"))
+	})
+})

mirrors.go

@@ -5,9 +5,12 @@ import (
 	"encoding/json"
 	"github.com/go-chi/chi/v5"
 	"net/http"
+	"strconv"
 	"strings"
 )
 
+// legacyMirrorsHandler will list the mirrors by region in the legacy format
+// it is preferred to use mirrors.json, but this handler is here for build support
 func legacyMirrorsHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 
@@ -26,6 +29,7 @@ func legacyMirrorsHandler(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(mirrorOutput)
 }
 
+// mirrorsHandler is a simple handler that will return the list of servers
 func mirrorsHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(servers)
@@ -42,10 +46,13 @@ var (
 	statusUnknown []byte
 )
 
+// mirrorStatusHandler is a fancy svg-returning handler.
+// it is used to display mirror statuses on a config repo of sorts
 func mirrorStatusHandler(w http.ResponseWriter, r *http.Request) {
 	serverHost := chi.URLParam(r, "server")
 
 	w.Header().Set("Content-Type", "image/svg+xml;charset=utf-8")
+	w.Header().Set("Cache-Control", "max-age=120")
 
 	if serverHost == "" {
 		w.Write(statusUnknown)
@@ -57,13 +64,31 @@ func mirrorStatusHandler(w http.ResponseWriter, r *http.Request) {
 	server, ok := hostMap[serverHost]
 
 	if !ok {
+		w.Header().Set("Content-Length", strconv.Itoa(len(statusUnknown)))
 		w.Write(statusUnknown)
 		return
 	}
 
+	key := "offline"
+
 	if server.Available {
+		key = "online"
+	}
+
+	w.Header().Set("ETag", "\""+key+"\"")
+
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Trim(match, "\"") == key {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	if server.Available {
+		w.Header().Set("Content-Length", strconv.Itoa(len(statusUp)))
 		w.Write(statusUp)
 	} else {
+		w.Header().Set("Content-Length", strconv.Itoa(len(statusDown)))
 		w.Write(statusDown)
 	}
 }

servers.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"crypto/tls"
 	"github.com/jmcvetta/randutil"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
@@ -20,12 +21,15 @@ var (
 		},
 	}
 
+	checkTLSConfig *tls.Config = nil
+
 	checks = []serverCheck{
 		checkHttp,
 		checkTLS,
 	}
 )
 
+// Server represents a download server
 type Server struct {
 	Available  bool               `json:"available"`
 	Host       string             `json:"host"`