From 290e5575cc4907565908137c8b67c6cc6a64d9dd Mon Sep 17 00:00:00 2001 From: Tyler Date: Sat, 6 Aug 2022 16:19:12 -0400 Subject: [PATCH 01/10] Initial testing implementation of checks --- check.go | 127 +++++++++++++++++++++++++++++++++++++++++++++++++++++ config.go | 3 ++ servers.go | 112 +++++++++++++++++----------------------------- 3 files changed, 171 insertions(+), 71 deletions(-) create mode 100644 check.go diff --git a/check.go b/check.go new file mode 100644 index 0000000..f809b6b --- /dev/null +++ b/check.go @@ -0,0 +1,127 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + log "github.com/sirupsen/logrus" + "net/http" + "net/url" + "runtime" + "time" +) + +var ( + ErrHttpsRedirect = errors.New("unexpected forced https redirect") + ErrCertExpired = errors.New("certificate is expired") +) + +// checkHttp checks a URL for validity, and checks redirects +func checkHttp(server *Server, logFields log.Fields) (bool, error) { + u := &url.URL{ + Scheme: "http", + Host: server.Host, + Path: server.Path, + } + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + + req.Header.Set("User-Agent", "ArmbianRouter/1.0 (Go "+runtime.Version()+")") + + if err != nil { + return false, err + } + + res, err := checkClient.Do(req) + + if err != nil { + return false, err + } + + logFields["responseCode"] = res.StatusCode + + if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound || res.StatusCode == http.StatusNotFound { + if res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound { + location := res.Header.Get("Location") + + logFields["url"] = location + + // Check that we don't redirect to https from a http url + if u.Scheme == "http" { + res, err := checkRedirect(location) + + if !res || err != nil { + return res, err + } + } + } + + return true, nil + } + + logFields["cause"] = fmt.Sprintf("Unexpected http status %d", res.StatusCode) + + return false, nil +} + +// checkRedirect parses a location header response and checks the scheme +func checkRedirect(locationHeader string) (bool, error) { + newUrl, err := url.Parse(locationHeader) + + if err != nil { + return false, err + } + + if newUrl.Scheme == "https" { + return false, ErrHttpsRedirect + } + + return true, nil +} + +// checkTLS checks tls certificates from a host, ensures they're valid, and not expired. +func checkTLS(server *Server, logFields log.Fields) (bool, error) { + conn, err := tls.Dial("tcp", server.Host+":443", nil) + + if err != nil { + return false, err + } + + defer conn.Close() + + err = conn.VerifyHostname(server.Host) + + if err != nil { + return false, err + } + + now := time.Now() + + state := conn.ConnectionState() + + opts := x509.VerifyOptions{ + CurrentTime: time.Now(), + } + + for _, cert := range state.PeerCertificates { + if _, err := cert.Verify(opts); err != nil { + logFields["peerCert"] = cert.Subject.String() + return false, err + } + if now.Before(cert.NotBefore) || now.After(cert.NotAfter) { + return false, err + } + } + + for _, chain := range state.VerifiedChains { + for _, cert := range chain { + if now.Before(cert.NotBefore) || now.After(cert.NotAfter) { + logFields["cert"] = cert.Subject.String() + return false, ErrCertExpired + } + } + } + + return true, nil +} diff --git a/config.go b/config.go index 1b17844..ab7171b 100644 --- a/config.go +++ b/config.go @@ -39,6 +39,9 @@ func reloadConfig() { serverCache.Resize(viper.GetInt("cacheSize")) } + // Purge the cache to ensure we don't have any invalid servers in it + serverCache.Purge() + // Set top choice count topChoices = viper.GetInt("topChoices") diff --git a/servers.go b/servers.go index 26aecaa..5b2c1ed 100644 --- a/servers.go +++ b/servers.go @@ -7,10 +7,7 @@ import ( "math" "net" "net/http" - "net/url" - "runtime" "sort" - "strings" "sync" "time" ) @@ -18,6 +15,14 @@ import ( var ( checkClient = &http.Client{ Timeout: 20 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + checks = []serverCheck{ + checkHttp, + checkTLS, } ) @@ -33,87 +38,45 @@ type Server struct { LastChange time.Time `json:"lastChange"` } +type serverCheck func(server *Server, logFields log.Fields) (bool, error) + +// checkStatus runs all status checks against a server func (server *Server) checkStatus() { - req, err := http.NewRequest(http.MethodGet, "http://"+server.Host+"/"+strings.TrimLeft(server.Path, "/"), nil) - - req.Header.Set("User-Agent", "ArmbianRouter/1.0 (Go "+runtime.Version()+")") - - if err != nil { - // This should never happen. - log.WithFields(log.Fields{ - "server": server.Host, - "error": err, - }).Warning("Invalid request! This should not happen, please check config.") - return + logFields := log.Fields{ + "host": server.Host, } - res, err := checkClient.Do(req) + var res bool + var err error - if err != nil { + for _, check := range checks { + res, err = check(server, logFields) + + if err != nil { + logFields["error"] = err + } + + if !res { + break + } + } + + if !res { if server.Available { - log.WithFields(log.Fields{ - "server": server.Host, - "error": err, - }).Info("Server went offline") + log.WithFields(logFields).Info("Server went offline") server.Available = false server.LastChange = time.Now() } else { - log.WithFields(log.Fields{ - "server": server.Host, - "error": err, - }).Debug("Server is still offline") + log.WithFields(logFields).Debug("Server is still offline") } + return - } - - responseFields := log.Fields{ - "server": server.Host, - "responseCode": res.StatusCode, - } - - if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound || res.StatusCode == http.StatusNotFound { - if res.StatusCode == http.StatusMovedPermanently || res.StatusCode == http.StatusFound { - location := res.Header.Get("Location") - - responseFields["url"] = location - - log.WithFields(responseFields).Debug("Server responded with redirect") - - newUrl, err := url.Parse(location) - - if err != nil { - if server.Available { - log.WithFields(responseFields).Warning("Server returned invalid url") - server.Available = false - server.LastChange = time.Now() - } - return - } - - if newUrl.Scheme == "https" { - if server.Available { - responseFields["url"] = location - log.WithFields(responseFields).Warning("Server returned https url for http request") - server.Available = false - server.LastChange = time.Now() - } - return - } - } - + } else { if !server.Available { server.Available = true server.LastChange = time.Now() - log.WithFields(responseFields).Info("Server is online") - } - } else { - log.WithFields(responseFields).Debug("Server status not known") - - if server.Available { - log.WithFields(responseFields).Info("Server went offline") - server.Available = false - server.LastChange = time.Now() + log.WithFields(logFields).Info("Server is online") } } } @@ -223,6 +186,13 @@ func (s ServerList) Closest(ip net.IP) (*Server, float64, error) { dist := choice.Item.(ComputedDistance) + if !dist.Server.Available { + // Choose a new server and refresh cache + serverCache.Remove(ip.String()) + + return s.Closest(ip) + } + return dist.Server, dist.Distance, nil } @@ -233,7 +203,7 @@ func hsin(theta float64) float64 { // Distance function returns the distance (in meters) between two points of // a given longitude and latitude relatively accurately (using a spherical -// approximation of the Earth) through the Haversin Distance Formula for +// approximation of the Earth) through the Haversine Distance Formula for // great arc distance on a sphere with accuracy for small distances // // point coordinates are supplied in degrees and converted into rad. in the func -- 2.40.1 From ffcb068ff5793d950ba60cc0bc21ccbac41626da Mon Sep 17 00:00:00 2001 From: Tyler Date: Sat, 6 Aug 2022 16:20:46 -0400 Subject: [PATCH 02/10] Initial testing implementation of checks --- .drone.yml | 12 +++++-- .gitignore | 3 +- LICENSE | 2 +- README.md | 18 +++++++++++ armbianmirror_suite_test.go | 13 ++++++++ check_test.go | 64 +++++++++++++++++++++++++++++++++++++ go.mod | 9 ++++-- go.sum | 56 +++++++++++++++++++++++++++++--- 8 files changed, 166 insertions(+), 11 deletions(-) create mode 100644 README.md create mode 100644 armbianmirror_suite_test.go create mode 100644 check_test.go diff --git a/.drone.yml b/.drone.yml index 041b6ec..8f4a4d9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3,6 +3,14 @@ name: default type: docker steps: + - name: test + image: golang:alpine + volumes: + - name: build + path: /build + commands: + - go mod download + - ginkgo . - name: build image: tystuyfzand/goc:latest volumes: @@ -16,7 +24,7 @@ steps: environment: GOOS: linux,windows,darwin GOARCH: 386,amd64,arm,arm64 - depends_on: [ clone ] + depends_on: [ test ] - name: release image: plugins/gitea-release volumes: @@ -48,7 +56,7 @@ steps: from_secret: docker_password repo: registry.meow.tf/tyler/armbian-router registry: registry.meow.tf - depends_on: [ clone ] + depends_on: [ test ] when: event: tag volumes: diff --git a/.gitignore b/.gitignore index 4f71822..9e4a423 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ userdata.csv dlrouter-apt.yaml *.yaml -!dlrouter.yaml \ No newline at end of file +!dlrouter.yaml +*.exe \ No newline at end of file diff --git a/LICENSE b/LICENSE index d14ad8e..6e84aab 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2022 Tyler Stuyfzand +Copyright (c) 2022 Tyler Stuyfzand , Armbian Project Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1cdd564 --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +Armbian Redirector +================== + +This repository contains a redirect service for Armbian downloads, apt, etc. + +It uses multiple current technologies and best practices, including: + +- Go 1.17/1.18 +- GeoIP + Distance routing +- Server weighting, pooling (top x servers are served instead of a single one) +- Health checks (HTTP, TLS) + +Code Quality +------------ + +The code quality isn't the greatest/top tier. All code lives in the "main" package and should be moved at some point. + +Regardless, it is meant to be simple and easy to understand. \ No newline at end of file diff --git a/armbianmirror_suite_test.go b/armbianmirror_suite_test.go new file mode 100644 index 0000000..04b1730 --- /dev/null +++ b/armbianmirror_suite_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestArmbianMirror(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ArmbianMirror Suite") +} diff --git a/check_test.go b/check_test.go new file mode 100644 index 0000000..2418498 --- /dev/null +++ b/check_test.go @@ -0,0 +1,64 @@ +package main + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + log "github.com/sirupsen/logrus" + "net/http" + "net/http/httptest" + "net/url" + "strings" +) + +var _ = Describe("Check suite", func() { + Context("HTTP Checks", func() { + var ( + httpServer *httptest.Server + server *Server + handler http.HandlerFunc + ) + BeforeEach(func() { + httpServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler(w, r) + })) + + u, err := url.Parse(httpServer.URL) + + if err != nil { + panic(err) + } + + server = &Server{ + Host: u.Host, + Path: u.Path, + } + }) + AfterEach(func() { + httpServer.Close() + }) + It("Should successfully check for connectivity", func() { + handler = func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + } + + res, err := checkHttp(server, log.Fields{}) + + Expect(res).To(BeTrue()) + Expect(err).To(BeNil()) + }) + It("Should return an error when redirected to https", func() { + handler = func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", strings.Replace(httpServer.URL, "http://", "https://", -1)) + w.WriteHeader(http.StatusMovedPermanently) + } + + res, err := checkHttp(server, log.Fields{}) + + Expect(res).To(BeFalse()) + Expect(err).To(Equal(ErrHttpsRedirect)) + }) + }) + Context("TLS Checks", func() { + + }) +}) diff --git a/go.mod b/go.mod index 38c545f..87e53c5 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ require ( github.com/go-chi/chi/v5 v5.0.7 github.com/hashicorp/golang-lru v0.5.4 github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff + github.com/onsi/ginkgo/v2 v2.1.4 + github.com/onsi/gomega v1.20.0 github.com/oschwald/maxminddb-golang v1.8.0 github.com/prometheus/client_golang v1.11.0 github.com/sirupsen/logrus v1.8.1 @@ -18,6 +20,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.8 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect @@ -31,9 +34,11 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect - golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect + golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect golang.org/x/text v0.3.7 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ce8227a..ef02281 100644 --- a/go.sum +++ b/go.sum @@ -110,6 +110,8 @@ github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -126,6 +128,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -175,8 +178,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -193,6 +197,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -232,6 +237,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -293,6 +299,21 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= +github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk= github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -368,6 +389,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= @@ -391,6 +413,7 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -427,8 +450,10 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -452,6 +477,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -465,8 +491,13 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -498,6 +529,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -509,11 +541,14 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -540,6 +575,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -564,10 +600,15 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -627,6 +668,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -635,10 +677,10 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -780,8 +822,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -789,8 +832,10 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -801,8 +846,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -- 2.40.1 From 91b99572c2e34af8ba98181983ed32d78f6625d9 Mon Sep 17 00:00:00 2001 From: Tyler Date: Sat, 6 Aug 2022 16:22:51 -0400 Subject: [PATCH 03/10] Ensure ginkgo is installed --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index 8f4a4d9..1d9584b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -10,6 +10,7 @@ steps: path: /build commands: - go mod download + - go install github.com/onsi/ginkgo/v2/ginkgo - ginkgo . - name: build image: tystuyfzand/goc:latest -- 2.40.1 From 8ea77adee29523ba50bf8b4cb6b77a523226af09 Mon Sep 17 00:00:00 2001 From: Tyler Date: Sat, 6 Aug 2022 16:23:50 -0400 Subject: [PATCH 04/10] Update go.sum --- go.sum | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go.sum b/go.sum index ef02281..7381e69 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -197,6 +198,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -677,6 +679,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -- 2.40.1 From 08da75d3093adb741bae521e8d13f8169008c7d7 Mon Sep 17 00:00:00 2001 From: Tyler Date: Sat, 6 Aug 2022 16:24:45 -0400 Subject: [PATCH 05/10] Disable cgo --- .drone.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index 1d9584b..2e70193 100644 --- a/.drone.yml +++ b/.drone.yml @@ -12,6 +12,8 @@ steps: - go mod download - go install github.com/onsi/ginkgo/v2/ginkgo - ginkgo . + environment: + CGO_ENABLED: '0'' - name: build image: tystuyfzand/goc:latest volumes: -- 2.40.1 From 3c5656284cf0df944f72e7cb34e135ee2c06ccc7 Mon Sep 17 00:00:00 2001 From: Tyler Date: Sat, 6 Aug 2022 16:25:05 -0400 Subject: [PATCH 06/10] Fix extra character on cgo environment variable --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 2e70193..d3be0c4 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,7 +13,7 @@ steps: - go install github.com/onsi/ginkgo/v2/ginkgo - ginkgo . environment: - CGO_ENABLED: '0'' + CGO_ENABLED: '0' - name: build image: tystuyfzand/goc:latest volumes: -- 2.40.1 From 3f71aced930fccf7533ee620a4b78692b285af39 Mon Sep 17 00:00:00 2001 From: Tyler Date: Sun, 14 Aug 2022 03:42:49 -0400 Subject: [PATCH 07/10] Improve test coverage and documentation --- check.go | 9 ++- check_test.go | 182 ++++++++++++++++++++++++++++++++++++++++++++------ config.go | 38 +++++++---- http.go | 36 +++++++++- main.go | 10 ++- map.go | 8 ++- map_test.go | 16 +++++ mirrors.go | 25 +++++++ servers.go | 4 ++ 9 files changed, 287 insertions(+), 41 deletions(-) create mode 100644 map_test.go diff --git a/check.go b/check.go index f809b6b..afdb0fd 100644 --- a/check.go +++ b/check.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" log "github.com/sirupsen/logrus" + "net" "net/http" "net/url" "runtime" @@ -82,7 +83,13 @@ func checkRedirect(locationHeader string) (bool, error) { // checkTLS checks tls certificates from a host, ensures they're valid, and not expired. func checkTLS(server *Server, logFields log.Fields) (bool, error) { - conn, err := tls.Dial("tcp", server.Host+":443", nil) + host, port, err := net.SplitHostPort(server.Host) + + if port == "" { + port = "443" + } + + conn, err := tls.Dial("tcp", host+":"+port, checkTLSConfig) if err != nil { return false, err diff --git a/check_test.go b/check_test.go index 2418498..7b4c758 100644 --- a/check_test.go +++ b/check_test.go @@ -1,40 +1,88 @@ package main import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" log "github.com/sirupsen/logrus" + "math/big" + "net" "net/http" "net/http/httptest" "net/url" "strings" + "time" ) +func genTestCerts(notBefore, notAfter time.Time) (*pem.Block, *pem.Block, error) { + // Create a Certificate Authority Cert + template := x509.Certificate{ + SerialNumber: big.NewInt(0), + Subject: pkix.Name{CommonName: "localhost"}, + SignatureAlgorithm: x509.SHA256WithRSA, + NotBefore: notBefore, + NotAfter: notAfter, + BasicConstraintsValid: true, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyAgreement | x509.KeyUsageKeyEncipherment | x509.KeyUsageDataEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + + // Create a Private Key + key, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, fmt.Errorf("Could not generate rsa key - %s", err) + } + + // Use CA Cert to sign a CSR and create a Public Cert + csr := &key.PublicKey + cert, err := x509.CreateCertificate(rand.Reader, &template, &template, csr, key) + if err != nil { + return nil, nil, fmt.Errorf("Could not generate certificate - %s", err) + } + + // Convert keys into pem.Block + c := &pem.Block{Type: "CERTIFICATE", Bytes: cert} + k := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)} + return c, k, nil +} + var _ = Describe("Check suite", func() { + var ( + httpServer *httptest.Server + server *Server + handler http.HandlerFunc + ) + BeforeEach(func() { + httpServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler(w, r) + })) + }) + AfterEach(func() { + httpServer.Close() + }) + setupServer := func() { + u, err := url.Parse(httpServer.URL) + + if err != nil { + panic(err) + } + server = &Server{ + Host: u.Host, + Path: u.Path, + } + } + Context("HTTP Checks", func() { - var ( - httpServer *httptest.Server - server *Server - handler http.HandlerFunc - ) BeforeEach(func() { - httpServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - handler(w, r) - })) - - u, err := url.Parse(httpServer.URL) - - if err != nil { - panic(err) - } - - server = &Server{ - Host: u.Host, - Path: u.Path, - } - }) - AfterEach(func() { - httpServer.Close() + httpServer.Start() + setupServer() }) It("Should successfully check for connectivity", func() { handler = func(w http.ResponseWriter, r *http.Request) { @@ -59,6 +107,96 @@ var _ = Describe("Check suite", func() { }) }) Context("TLS Checks", func() { + var ( + x509Cert *x509.Certificate + ) + setupCerts := func(notBefore, notAfter time.Time) { + cert, key, err := genTestCerts(notBefore, notAfter) + if err != nil { + panic("Unable to generate test certs") + } + + x509Cert, err = x509.ParseCertificate(cert.Bytes) + + if err != nil { + panic("Unable to parse certificate from bytes: " + err.Error()) + } + + tlsPair, err := tls.X509KeyPair(pem.EncodeToMemory(cert), pem.EncodeToMemory(key)) + + if err != nil { + panic("Unable to load tls key pair: " + err.Error()) + } + + httpServer.TLS = &tls.Config{ + Certificates: []tls.Certificate{tlsPair}, + } + + httpServer.StartTLS() + setupServer() + } + Context("CA Tests", func() { + BeforeEach(func() { + setupCerts(time.Now(), time.Now().Add(24*time.Hour)) + }) + It("Should fail due to invalid ca", func() { + res, err := checkTLS(server, log.Fields{}) + + Expect(res).To(BeFalse()) + Expect(err).ToNot(BeNil()) + }) + It("Should successfully validate certificates (valid ca, valid date/times, etc)", func() { + pool := x509.NewCertPool() + + pool.AddCert(x509Cert) + + checkTLSConfig = &tls.Config{RootCAs: pool} + + res, err := checkTLS(server, log.Fields{}) + + Expect(res).To(BeFalse()) + Expect(err).ToNot(BeNil()) + + checkTLSConfig = nil + }) + }) + Context("Expiration tests", func() { + AfterEach(func() { + checkTLSConfig = nil + }) + It("Should fail due to not yet valid certificate", func() { + setupCerts(time.Now().Add(5*time.Hour), time.Now().Add(10*time.Hour)) + + // Trust our certs + pool := x509.NewCertPool() + + pool.AddCert(x509Cert) + + checkTLSConfig = &tls.Config{RootCAs: pool} + + // Check TLS + res, err := checkTLS(server, log.Fields{}) + + Expect(res).To(BeFalse()) + Expect(err).ToNot(BeNil()) + }) + It("Should fail due to expired certificate", func() { + setupCerts(time.Now().Add(-10*time.Hour), time.Now().Add(-5*time.Hour)) + + // Trust our certs + pool := x509.NewCertPool() + + pool.AddCert(x509Cert) + + checkTLSConfig = &tls.Config{RootCAs: pool} + + // Check TLS + res, err := checkTLS(server, log.Fields{}) + + Expect(res).To(BeFalse()) + Expect(err).ToNot(BeNil()) + }) + }) }) }) diff --git a/config.go b/config.go index ab7171b..216eaf9 100644 --- a/config.go +++ b/config.go @@ -3,6 +3,7 @@ package main import ( lru "github.com/hashicorp/golang-lru" "github.com/oschwald/maxminddb-golang" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" log "github.com/sirupsen/logrus" @@ -13,13 +14,13 @@ import ( "sync" ) -func reloadConfig() { +func reloadConfig() error { log.Info("Loading configuration...") err := viper.ReadInConfig() // Find and read the config file if err != nil { // Handle errors reading the config file - log.WithError(err).Fatalln("Unable to load config file") + return errors.Wrap(err, "Unable to read configuration") } // db will never be reloaded. @@ -28,7 +29,7 @@ func reloadConfig() { db, err = maxminddb.Open(viper.GetString("geodb")) if err != nil { - log.WithError(err).Fatalln("Unable to open database") + return errors.Wrap(err, "Unable to open database") } } @@ -46,10 +47,14 @@ func reloadConfig() { topChoices = viper.GetInt("topChoices") // Reload map file - reloadMap() + if err := reloadMap(); err != nil { + return errors.Wrap(err, "Unable to load map file") + } // Reload server list - reloadServers() + if err := reloadServers(); err != nil { + return errors.Wrap(err, "Unable to load servers") + } // Create mirror map mirrors := make(map[string][]*Server) @@ -77,11 +82,16 @@ func reloadConfig() { // Force check go servers.Check() + + return nil } -func reloadServers() { +func reloadServers() error { var serverList []ServerConfig - viper.UnmarshalKey("servers", &serverList) + + if err := viper.UnmarshalKey("servers", &serverList); err != nil { + return err + } var wg sync.WaitGroup @@ -109,7 +119,7 @@ func reloadServers() { "error": err, "server": server, }).Warning("Server is invalid") - return + return err } hosts[u.Host] = true @@ -161,6 +171,8 @@ func reloadServers() { servers = append(servers[:i], servers[i+1:]...) } + + return nil } var metricReplacer = strings.NewReplacer(".", "_", "-", "_") @@ -217,20 +229,22 @@ func addServer(server ServerConfig, u *url.URL) *Server { return s } -func reloadMap() { +func reloadMap() error { mapFile := viper.GetString("dl_map") if mapFile == "" { - return + return nil } log.WithField("file", mapFile).Info("Loading download map") - newMap, err := loadMap(mapFile) + newMap, err := loadMapFile(mapFile) if err != nil { - return + return err } dlMap = newMap + + return nil } diff --git a/http.go b/http.go index fd47234..ec8f511 100644 --- a/http.go +++ b/http.go @@ -13,11 +13,18 @@ import ( "strings" ) +// statusHandler is a simple handler that will always return 200 OK with a body of "OK" func statusHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte("OK")) + + if r.Method != http.MethodHead { + w.Write([]byte("OK")) + } } +// redirectHandler is the default "not found" handler which handles redirects +// if the environment variable OVERRIDE_IP is set, it will use that ip address +// this is useful for local testing when you're on the local network func redirectHandler(w http.ResponseWriter, r *http.Request) { ipStr, _, err := net.SplitHostPort(r.RemoteAddr) @@ -41,6 +48,8 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) { var server *Server var distance float64 + // If the path has a prefix of region/NA, it will use specific regions instead + // of the default geographical distance if strings.HasPrefix(r.URL.Path, "/region") { parts := strings.Split(r.URL.Path, "/") @@ -72,6 +81,7 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) { } } + // If none of the above exceptions are matched, we use the geographical distance based on IP if server == nil { server, distance, err = servers.Closest(ip) @@ -81,14 +91,19 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) { } } + // If we don't have a scheme, we'll use https by default scheme := r.URL.Scheme if scheme == "" { scheme = "https" } + // redirectPath is a combination of server path (which can be something like /armbian) + // and the URL path. + // Example: /armbian + /some/path = /armbian/some/path redirectPath := path.Join(server.Path, r.URL.Path) + // If we have a dlMap, we map the url to a final path instead if dlMap != nil { if newPath, exists := dlMap[strings.TrimLeft(r.URL.Path, "/")]; exists { downloadsMapped.Inc() @@ -100,6 +115,7 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) { redirectPath += "/" } + // We need to build the final url now u := &url.URL{ Scheme: scheme, Host: server.Host, @@ -109,6 +125,7 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) { server.Redirects.Inc() redirectsServed.Inc() + // If we used geographical distance, we add an X-Geo-Distance header for debug. if distance > 0 { w.Header().Set("X-Geo-Distance", fmt.Sprintf("%f", distance)) } @@ -117,7 +134,16 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusFound) } +// reloadHandler is an http handler which lets us reload the server configuration +// It is only enabled when the reloadToken is set in the configuration func reloadHandler(w http.ResponseWriter, r *http.Request) { + expectedToken := viper.GetString("reloadToken") + + if expectedToken == "" { + w.WriteHeader(http.StatusUnauthorized) + return + } + token := r.Header.Get("Authorization") if token == "" || !strings.HasPrefix(token, "Bearer") || !strings.Contains(token, " ") { @@ -127,12 +153,16 @@ func reloadHandler(w http.ResponseWriter, r *http.Request) { token = token[strings.Index(token, " ")+1:] - if token != viper.GetString("reloadToken") { + if token != expectedToken { w.WriteHeader(http.StatusUnauthorized) return } - reloadConfig() + if err := reloadConfig(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } w.WriteHeader(http.StatusOK) w.Write([]byte("OK")) diff --git a/main.go b/main.go index 9900b1c..9115c55 100644 --- a/main.go +++ b/main.go @@ -104,7 +104,9 @@ func main() { viper.SetConfigFile(*configFlag) } - reloadConfig() + if err := reloadConfig(); err != nil { + log.WithError(err).Fatalln("Unable to load configuration") + } // Start check loop go servers.checkLoop() @@ -143,6 +145,10 @@ func main() { break } - reloadConfig() + err := reloadConfig() + + if err != nil { + log.WithError(err).Warning("Did not reload configuration due to error") + } } } diff --git a/map.go b/map.go index f6019f9..265862d 100644 --- a/map.go +++ b/map.go @@ -7,7 +7,8 @@ import ( "strings" ) -func loadMap(file string) (map[string]string, error) { +// loadMapFile loads a file as a map +func loadMapFile(file string) (map[string]string, error) { f, err := os.Open(file) if err != nil { @@ -16,6 +17,11 @@ func loadMap(file string) (map[string]string, error) { defer f.Close() + return loadMap(f) +} + +// loadMap loads a pipe separated file of mappings +func loadMap(f io.Reader) (map[string]string, error) { m := make(map[string]string) r := csv.NewReader(f) diff --git a/map_test.go b/map_test.go new file mode 100644 index 0000000..c9fa3ca --- /dev/null +++ b/map_test.go @@ -0,0 +1,16 @@ +package main + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "strings" +) + +var _ = Describe("Map", func() { + It("Should successfully load the map", func() { + m, err := loadMap(strings.NewReader(`bananapi/Bullseye_current|bananapi/archive/Armbian_21.08.1_Bananapi_bullseye_current_5.10.60.img.xz|Aug 26 2021|332M`)) + + Expect(err).To(BeNil()) + Expect(m["bananapi/Bullseye_current"]).To(Equal("bananapi/archive/Armbian_21.08.1_Bananapi_bullseye_current_5.10.60.img.xz")) + }) +}) diff --git a/mirrors.go b/mirrors.go index ed92411..269da2b 100644 --- a/mirrors.go +++ b/mirrors.go @@ -5,9 +5,12 @@ import ( "encoding/json" "github.com/go-chi/chi/v5" "net/http" + "strconv" "strings" ) +// legacyMirrorsHandler will list the mirrors by region in the legacy format +// it is preferred to use mirrors.json, but this handler is here for build support func legacyMirrorsHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -26,6 +29,7 @@ func legacyMirrorsHandler(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(mirrorOutput) } +// mirrorsHandler is a simple handler that will return the list of servers func mirrorsHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(servers) @@ -42,10 +46,13 @@ var ( statusUnknown []byte ) +// mirrorStatusHandler is a fancy svg-returning handler. +// it is used to display mirror statuses on a config repo of sorts func mirrorStatusHandler(w http.ResponseWriter, r *http.Request) { serverHost := chi.URLParam(r, "server") w.Header().Set("Content-Type", "image/svg+xml;charset=utf-8") + w.Header().Set("Cache-Control", "max-age=120") if serverHost == "" { w.Write(statusUnknown) @@ -57,13 +64,31 @@ func mirrorStatusHandler(w http.ResponseWriter, r *http.Request) { server, ok := hostMap[serverHost] if !ok { + w.Header().Set("Content-Length", strconv.Itoa(len(statusUnknown))) w.Write(statusUnknown) return } + key := "offline" + if server.Available { + key = "online" + } + + w.Header().Set("ETag", "\""+key+"\"") + + if match := r.Header.Get("If-None-Match"); match != "" { + if strings.Trim(match, "\"") == key { + w.WriteHeader(http.StatusNotModified) + return + } + } + + if server.Available { + w.Header().Set("Content-Length", strconv.Itoa(len(statusUp))) w.Write(statusUp) } else { + w.Header().Set("Content-Length", strconv.Itoa(len(statusDown))) w.Write(statusDown) } } diff --git a/servers.go b/servers.go index 5b2c1ed..425936c 100644 --- a/servers.go +++ b/servers.go @@ -1,6 +1,7 @@ package main import ( + "crypto/tls" "github.com/jmcvetta/randutil" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -20,12 +21,15 @@ var ( }, } + checkTLSConfig *tls.Config = nil + checks = []serverCheck{ checkHttp, checkTLS, } ) +// Server represents a download server type Server struct { Available bool `json:"available"` Host string `json:"host"` -- 2.40.1 From c4bd02485c332b01e95c0ea5a5f5f19349a073cf Mon Sep 17 00:00:00 2001 From: Tyler Date: Sun, 14 Aug 2022 03:49:28 -0400 Subject: [PATCH 08/10] Fix tests and add coverage --- .drone.yml | 3 ++- go.mod | 1 + go.sum | 4 +--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index d3be0c4..0e1ee02 100644 --- a/.drone.yml +++ b/.drone.yml @@ -11,7 +11,8 @@ steps: commands: - go mod download - go install github.com/onsi/ginkgo/v2/ginkgo - - ginkgo . + - ginkgo --randomize-all --p --cover --coverprofile=cover.out . + - go tool cover -func=cover.out environment: CGO_ENABLED: '0' - name: build diff --git a/go.mod b/go.mod index 87e53c5..08766c5 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.20.0 github.com/oschwald/maxminddb-golang v1.8.0 + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/viper v1.10.1 diff --git a/go.sum b/go.sum index 7381e69..ebc3e6b 100644 --- a/go.sum +++ b/go.sum @@ -128,7 +128,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -198,7 +197,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -324,6 +322,7 @@ github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhEC github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -679,7 +678,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -- 2.40.1 From 5ff4aa9faebff251c1c2883a82ed379bcd50540b Mon Sep 17 00:00:00 2001 From: Tyler Date: Sun, 14 Aug 2022 03:50:11 -0400 Subject: [PATCH 09/10] go.sum magically removed ginkgo --- go.sum | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go.sum b/go.sum index ebc3e6b..1cb0b01 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -197,6 +198,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -678,6 +680,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -- 2.40.1 From 2f71e97f2e1c1f9736d8b8f2dc923c8ec4d4b53b Mon Sep 17 00:00:00 2001 From: Tyler Date: Sun, 14 Aug 2022 04:02:10 -0400 Subject: [PATCH 10/10] Improve readme with proper information and configuration --- README.md | 102 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1cdd564..3c32821 100644 --- a/README.md +++ b/README.md @@ -15,4 +15,104 @@ Code Quality The code quality isn't the greatest/top tier. All code lives in the "main" package and should be moved at some point. -Regardless, it is meant to be simple and easy to understand. \ No newline at end of file +Regardless, it is meant to be simple and easy to understand. + +Configuration +------------- + +### Modes + +#### Redirect + +Standard redirect functionality + +#### Download Mapping + +Uses the `dl_map` configuration variable to enable mapping of paths to new paths. + +Think symlinks, but in a generated file. + +### Mirrors +Mirror targets with trailing slash are placed in the yaml configuration file. + +### Example YAML +```yaml +# GeoIP Database Path +geodb: GeoLite2-City.mmdb + +# Comment out to disable +dl_map: userdata.csv + +# LRU Cache Size (in items) +cacheSize: 1024 + +# Server definition +# Weights are just like nginx, where if it's > 1 it'll be chosen x out of x + total times +# By default, the top 3 servers are used for choosing the best. +# server = full url or host+path +# weight = int +# optional: latitude, longitude (float) +servers: + - server: armbian.12z.eu/apt/ + - server: armbian.chi.auroradev.org/apt/ + weight: 15 + latitude: 41.8879 + longitude: -88.1995 +```` + +## API + +`/status` + +Meant for a simple health check (nginx/etc can 502 or similar if down) + +`/reload` + +Flushes cache and reloads configuration and mapping. Requires reloadToken to be set in the configuration, and a matching token provided in `Authorization: Bearer TOKEN` + +`/mirrors` + +Shows all mirrors in the legacy (by region) format + +`/mirrors.json` + +Shows all mirrors in the new JSON format. Example: + +```json +[ + { + "available":true, + "host":"imola.armbian.com", + "path":"/apt/", + "latitude":46.0503, + "longitude":14.5046, + "weight":10, + "continent":"EU", + "lastChange":"2022-08-12T06:52:35.029565986Z" + } +] +``` + +`/mirrors/{server}.svg` + +Magic SVG path to show badges based on server status, for use in dynamic mirror lists. + +`/dl_map` + +Shows json-encoded download mappings + +`/geoip` + +Shows GeoIP information for the requester + +`/region/REGIONCODE/PATH` + +Using this magic path will redirect to the desired region: + +* NA - North America +* EU - Europe +* AS - Asia + +`/metrics` + +Prometheus metrics endpoint. Metrics aren't considered private, thus are exposed to the public. \ No newline at end of file -- 2.40.1