Caching of packages, new config format

.gitignore

@@ -1,5 +1,6 @@
 /deb-simple
+/simple
 /repo
 vendor
-conf.json
+conf.ini
 *.pprof

sample_conf.ini

@@ -1,12 +1,13 @@
-{
+[http]
-    "listenPort" : "9090",
+port = 9090
-    "rootRepoPath" : "/opt/deb-simple/repo",
+key = "abcdefg"
-    "supportedArch" : ["all","i386","amd64"],
+ssl = false
-    "distroNames":["stable"],
+
-    "pgpSecretKey": "secring.gpg",
+[repo]
-    "pgpPassphrase" : "",
+root = "repo/"
-    "enableSSL" : false,
+distros = "stable"
-    "SSLcert" : "server.crt",
+architectures = "all,i386,amd64"
-    "SSLkey" : "server.key",
+
-    "key" : "abcdefg"
+[pgp]
-}
+key = ""
+passphrase = ""

src/meow.tf/deb-simple/config.go

@@ -1 +1,56 @@
 package simple
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+type Conf struct {
+	Http HttpConf `ini:"http"`
+	Repo RepoConf `ini:"repo"`
+	PGP PGPConf `ini:"pgp"`
+}
+
+type HttpConf struct {
+	Port int `ini:"port"`
+	Key string `ini:"key"`
+	SSL bool `ini:"ssl"`
+	SSLCert string `ini:"cert"`
+	SSLKey string `ini:"key"`
+}
+
+type RepoConf struct {
+	Root string `ini:"root"`
+	Distros string `ini:"distros"`
+	Architectures string `ini:"architectures"`
+}
+
+type PGPConf struct {
+	Key string `ini:"key"`
+	Passphrase string `ini:"passphrase"`
+}
+
+func (c Conf) DistPath(distro string) string {
+	return filepath.Join(c.Repo.Root, "dists", distro)
+}
+
+func (c Conf) ArchPath(distro, arch string) string {
+	return filepath.Join(c.Repo.Root, "dists", distro, "main/binary-"+arch)
+}
+
+func (c Conf) PoolPath(distro, arch string) string {
+	return filepath.Join(c.Repo.Root, "pool/main", distro, arch)
+}
+
+func (c Conf) PoolPackagePath(distro, arch, name string) string {
+	name = packageName(name)
+	return filepath.Join(c.Repo.Root, "pool/main", distro, arch, name[0:1], name)
+}
+
+func (c RepoConf) DistroNames() []string {
+	return strings.Split(c.Distros, ",")
+}
+
+func (c RepoConf) ArchitectureNames() []string {
+	return strings.Split(c.Architectures, ",")
+}

src/meow.tf/deb-simple/glide.lock

@@ -1,10 +1,12 @@
-hash: 79d0f0e99e99ce9318bb0e8c6ab398657a20679dbe3c421c6fc588db7a738131
+hash: 33760adb1090508f6d56dfc1d600a664790a280e5fc4d84cb26a7e11ad0f1f35
-updated: 2017-06-11T03:04:15.102617-04:00
+updated: 2017-06-11T20:13:27.6668377-04:00
 imports:
 - name: github.com/blakesmith/ar
   version: 8bd4349a67f2533b078dbc524689d15dba0f4659
 - name: github.com/blang/semver
   version: b38d23b8782a487059e8fc8773e9a5b228a77cb6
+- name: github.com/go-ini/ini
+  version: d3de07a94d22b4a0972deb4b96d790c2c0ce8333
 - name: golang.org/x/crypto
   version: 7e9105388ebff089b3f99f0ef676ea55a6da3a7e
   subpackages:

src/meow.tf/deb-simple/glide.yaml

@@ -6,3 +6,5 @@ import:
   - openpgp
 - package: github.com/blang/semver
   version: ^3.5.0
+- package: github.com/go-ini/ini
+  version: ^1.28.0

src/meow.tf/deb-simple/http.go

@@ -11,214 +11,232 @@ import (
 	"github.com/blang/semver"
 )
 
-func uploadHandler(config Conf) http.Handler {
+func rescanHandler(w http.ResponseWriter, r *http.Request) {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	distroName := r.URL.Query().Get("distro")
-		if r.Method != "POST" {
-			http.Error(w, "method not supported", http.StatusMethodNotAllowed)
-			return
-		}
-		archType := r.URL.Query().Get("arch")
 
-		if archType == "" {
+	if distroName == "" {
-			archType = "all"
+		distroName = "stable"
-		}
+	}
 
-		distroName := r.URL.Query().Get("distro")
+	if _, exists := distros[distroName]; !exists {
+		httpErrorf(w, "Unable to find distro %s", distroName)
+		return
+	}
 
-		if distroName == "" {
+	distro := &Distro{Name: distroName, Architectures: make(map[string]map[string]*PackageFile)}
-			distroName = "stable"
-		}
 
-		key := r.URL.Query().Get("key")
+	scanInitialPackages(conf, distro)
 
-		if key == "" || key != config.Key {
+	distros[distroName] = distro
-			http.Error(w, "unauthorized", 403)
-			return
-		}
-
-		force := false
-
-		forceStr := r.URL.Query().Get("force")
-
-		if forceStr != "" && forceStr == "true" {
-			force = true
-		}
-
-		reader, err := r.MultipartReader()
-
-		if err != nil {
-			httpErrorf(w, "error creating multipart reader: %s", err)
-			return
-		}
-
-		mutex.RLock()
-
-		distro, exists := distros[distroName]
-
-		if !exists {
-			httpErrorf(w, "invalid distro: %s", distroName)
-			mutex.RUnlock()
-			return
-		}
-
-		packages, exists := distro.Architectures[archType]
-
-		if !exists {
-			httpErrorf(w, "invalid arch: %s", archType)
-			mutex.RUnlock()
-			return
-		}
-
-		mutex.RUnlock()
-
-		// Lock to prevent concurrent modification
-		mutex.Lock()
-		defer mutex.Unlock()
-
-		for {
-			part, err := reader.NextPart()
-
-			if err == io.EOF {
-				break
-			}
-
-			if part.FileName() == "" {
-				continue
-			}
-			newPath := config.PoolPackagePath(distroName, archType, part.FileName())
-
-			if _, err := os.Stat(newPath); err != nil {
-				if os.IsNotExist(err) {
-					if err := os.MkdirAll(newPath, 0755); err != nil {
-						httpErrorf(w, "error creating path: %s", err)
-						return
-					}
-				}
-			}
-
-			dst, err := os.Create(filepath.Join(newPath, part.FileName()))
-
-			if err != nil {
-				httpErrorf(w, "error creating deb file: %s", err)
-				return
-			}
-
-			defer dst.Close()
-
-			if _, err := io.Copy(dst, part); err != nil {
-				httpErrorf(w, "error writing deb file: %s", err)
-				return
-			}
-
-			// Get package name, if it already exists remove the old file.
-			f, err := newPackageFile(newPath, part.FileName())
-
-			if err != nil {
-				httpErrorf(w, "error loading package info: %s", err)
-			}
-
-			if p, exists := packages[f.Info.Package]; exists {
-				v1, err1 := semver.Parse(p.Info.Version)
-				v2, err2 := semver.Parse(f.Info.Version)
-
-				if err1 == nil && err2 == nil && v1.Compare(v2) > 0 && !force {
-					// Don't replace newer package
-					httpErrorf(w, "version in old package is greater than new: %s, %s - override with \"force\"", p.Info.Version, f.Info.Version)
-					return
-				}
-
-				// Archive old file
-				log.Println("Replacing", p.Name, "with", f.Name)
-
-				if err := os.Remove(p.Path); err != nil {
-					httpErrorf(w, "Unable to remove old package: %s", err)
-					return
-				}
-			}
-
-			packages[f.Info.Package] = f
-		}
-
-		log.Println("got lock, updating package list...")
-
-		// Recreate the package index and release file.
-
-		if err := createPackagesCached(config, distroName, archType, packages); err != nil {
-			httpErrorf(w, "error creating package: %s", err)
-			return
-		}
-
-		err = createRelease(config, distroName, archType)
-
-		if err != nil {
-			httpErrorf(w, "error creating package: %s", err)
-			return
-		}
-
-		w.WriteHeader(http.StatusOK)
-	})
 }
 
-func deleteHandler(config Conf) http.Handler {
+func uploadHandler(w http.ResponseWriter, r *http.Request) {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
-		if r.Method != "DELETE" {
+		http.Error(w, "method not supported", http.StatusMethodNotAllowed)
-			http.Error(w, "method not supported", http.StatusMethodNotAllowed)
+		return
-			return
+	}
-		}
-		var req DeleteObj
 
-		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+	archType := r.URL.Query().Get("arch")
-			httpErrorf(w, "failed to decode json: %s", err)
-			return
-		}
 
-		mutex.RLock()
+	if archType == "" {
+		archType = "all"
+	}
 
-		distro, exists := distros[req.DistroName]
+	distroName := r.URL.Query().Get("distro")
 
-		if !exists {
+	if distroName == "" {
-			httpErrorf(w, "invalid distro: %s", req.DistroName)
+		distroName = "stable"
-			mutex.RUnlock()
+	}
-			return
-		}
 
-		packages, exists := distro.Architectures[req.Arch]
+	key := r.URL.Query().Get("key")
 
-		if !exists {
+	if key == "" || key != conf.Http.Key {
-			httpErrorf(w, "invalid arch: %s", req.Arch)
+		http.Error(w, "unauthorized", 403)
-			mutex.RUnlock()
+		return
-			return
+	}
-		}
 
+	force := false
+
+	forceStr := r.URL.Query().Get("force")
+
+	if forceStr != "" && forceStr == "true" {
+		force = true
+	}
+
+	reader, err := r.MultipartReader()
+
+	if err != nil {
+		httpErrorf(w, "error creating multipart reader: %s", err)
+		return
+	}
+
+	mutex.RLock()
+
+	distro, exists := distros[distroName]
+
+	if !exists {
+		httpErrorf(w, "invalid distro: %s", distroName)
 		mutex.RUnlock()
+		return
+	}
 
-		debPath := filepath.Join(config.ArchPath(req.DistroName, req.Arch), req.Filename)
+	packages, exists := distro.Architectures[archType]
 
-		if err := os.Remove(debPath); err != nil {
+	if !exists {
-			httpErrorf(w, "failed to delete: %s", err)
+		httpErrorf(w, "invalid arch: %s", archType)
+		mutex.RUnlock()
+		return
+	}
+
+	mutex.RUnlock()
+
+	// Lock to prevent concurrent modification
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	for {
+		part, err := reader.NextPart()
+
+		if err == io.EOF {
+			break
+		}
+
+		if part.FileName() == "" {
+			continue
+		}
+		newPath := conf.PoolPackagePath(distroName, archType, part.FileName())
+
+		if _, err := os.Stat(newPath); err != nil {
+			if os.IsNotExist(err) {
+				if err := os.MkdirAll(newPath, 0755); err != nil {
+					httpErrorf(w, "error creating path: %s", err)
+					return
+				}
+			}
+		}
+
+		dst, err := os.Create(filepath.Join(newPath, part.FileName()))
+
+		if err != nil {
+			httpErrorf(w, "error creating deb file: %s", err)
 			return
 		}
 
-		key := r.URL.Query().Get("key")
+		defer dst.Close()
 
-		if key == "" || key != config.Key {
+		if _, err := io.Copy(dst, part); err != nil {
-			http.Error(w, "unauthorized", http.StatusForbidden)
+			httpErrorf(w, "error writing deb file: %s", err)
-			return
-		}
-		mutex.Lock()
-		defer mutex.Unlock()
-
-		log.Println("got lock, updating package list...")
-		if err := createPackagesCached(config, req.DistroName, req.Arch, packages); err != nil {
-			httpErrorf(w, "failed to delete package: %s", err)
 			return
 		}
 
-		if err := createRelease(config, req.DistroName, req.Arch); err != nil {
+		// Get package name, if it already exists remove the old file.
-			httpErrorf(w, "failed to delete package: %s", err)
+		f, err := newPackageFile(newPath, part.FileName())
-			return
+
+		if err != nil {
+			httpErrorf(w, "error loading package info: %s", err)
 		}
-	})
+
+		if p, exists := packages[f.Info.Package]; exists {
+			v1, err1 := semver.Parse(p.Info.Version)
+			v2, err2 := semver.Parse(f.Info.Version)
+
+			if err1 == nil && err2 == nil && v1.Compare(v2) > 0 && !force {
+				// Don't replace newer package
+				httpErrorf(w, "version in old package is greater than new: %s, %s - override with \"force\"", p.Info.Version, f.Info.Version)
+				return
+			}
+
+			// Archive old file
+			log.Println("Replacing", p.Name, "with", f.Name)
+
+			if err := os.Remove(p.Path); err != nil {
+				httpErrorf(w, "Unable to remove old package: %s", err)
+				return
+			}
+		}
+
+		packages[f.Info.Package] = f
+	}
+
+	log.Println("got lock, updating package list...")
+
+	// Recreate the package index and release file.
+
+	if err := createPackagesCached(conf, distroName, archType, packages); err != nil {
+		httpErrorf(w, "error creating package: %s", err)
+		return
+	}
+
+	err = createRelease(conf, distroName, archType)
+
+	if err != nil {
+		httpErrorf(w, "error creating package: %s", err)
+		return
+	}
+
+	w.WriteHeader(http.StatusCreated)
+}
+
+func deleteHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "DELETE" {
+		http.Error(w, "method not supported", http.StatusMethodNotAllowed)
+		return
+	}
+	var req DeleteObj
+
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		httpErrorf(w, "failed to decode json: %s", err)
+		return
+	}
+
+	mutex.RLock()
+
+	distro, exists := distros[req.DistroName]
+
+	if !exists {
+		httpErrorf(w, "invalid distro: %s", req.DistroName)
+		mutex.RUnlock()
+		return
+	}
+
+	packages, exists := distro.Architectures[req.Arch]
+
+	if !exists {
+		httpErrorf(w, "invalid arch: %s", req.Arch)
+		mutex.RUnlock()
+		return
+	}
+
+	mutex.RUnlock()
+
+	key := r.URL.Query().Get("key")
+
+	if key == "" || key != conf.Http.Key {
+		http.Error(w, "unauthorized", http.StatusForbidden)
+		return
+	}
+
+	debPath := filepath.Join(conf.ArchPath(req.DistroName, req.Arch), req.Filename)
+
+	if err := os.Remove(debPath); err != nil {
+		httpErrorf(w, "failed to delete: %s", err)
+		return
+	}
+
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	if err := createPackagesCached(conf, req.DistroName, req.Arch, packages); err != nil {
+		httpErrorf(w, "failed to delete package: %s", err)
+		return
+	}
+
+	if err := createRelease(conf, req.DistroName, req.Arch); err != nil {
+		httpErrorf(w, "failed to delete package: %s", err)
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
 }
 
 func httpErrorf(w http.ResponseWriter, format string, a ...interface{}) {

src/meow.tf/deb-simple/packages.go

@@ -16,6 +16,7 @@ import (
 	"crypto/sha1"
 	"crypto/sha256"
 	"github.com/blang/semver"
+	"encoding/json"
 )
 
 var (
@@ -23,20 +24,20 @@ var (
 )
 
 type PackageFile struct {
-	Path string
+	Path string `json:"path"`
-	Name string
+	Name string `json:"name"`
-	Size int64
+	Size int64 `json:"size"`
-	MD5Hash string
+	MD5Hash string `json:"md5"`
-	SHA1Hash string
+	SHA1Hash string `json:"sha1"`
-	SHA256Hash string
+	SHA256Hash string `json:"sha256"`
-	ControlData string
+	ControlData string `json:"control"`
-	Info *Package
+	Info *Package `json:"info"`
 }
 
 type Package struct {
-	Package string
+	Package string `json:"package"`
-	Version string
+	Version string `json:"version"`
-	Architecture string
+	Architecture string `json:"architecture"`
 }
 
 func parsePackageData(ctlData string) *Package {
@@ -57,14 +58,50 @@ func parsePackageData(ctlData string) *Package {
 }
 
 type Distro struct {
-	Name string
+	Name string `json:"name"`
-	Architectures map[string]map[string]*PackageFile
+	Architectures map[string]map[string]*PackageFile `json:"architectures"`
 }
 
 var (
 	distros map[string]*Distro = make(map[string]*Distro)
 )
 
+func loadCache(dist string) error {
+	f, err := os.Open(filepath.Join(conf.DistPath(dist), "dist.json"))
+
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+
+	var distro Distro
+
+	if err := json.NewDecoder(f).Decode(&distro); err != nil {
+		return err
+	}
+
+	distros[dist] = &distro
+
+	return nil
+}
+
+func saveCache(dist *Distro) error {
+	f, err := os.Create(filepath.Join(conf.DistPath(dist.Name), "dist.json"))
+
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+
+	if err := json.NewEncoder(f).Encode(dist); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func buildPackageList(config Conf, distro, arch string) (map[string]*PackageFile, error) {
 	m := make(map[string]*PackageFile)
 

src/meow.tf/deb-simple/server.go

@@ -1,53 +1,22 @@
 package simple
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
-	"path/filepath"
 	"sync"
 	"strings"
 	"golang.org/x/crypto/openpgp"
 	"runtime"
 	"errors"
+	"github.com/go-ini/ini"
 )
 
 var VERSION string = "1.1.0"
 
-type Conf struct {
-	ListenPort    string   `json:"listenPort"`
-	RootRepoPath  string   `json:"rootRepoPath"`
-	SupportArch   []string `json:"supportedArch"`
-	DistroNames   []string `json:"distroNames"`
-	PGPSecretKey  string   `json:"pgpSecretKey"`
-	PGPPassphrase string   `json:"pgpPassphrase"`
-	EnableSSL     bool     `json:"enableSSL"`
-	SSLCert       string   `json:"SSLcert"`
-	SSLKey        string   `json:"SSLkey"`
-	Key           string   `json:"key"`
-}
-
-func (c Conf) DistPath(distro string) string {
-	return filepath.Join(c.RootRepoPath, "dists", distro)
-}
-
-func (c Conf) ArchPath(distro, arch string) string {
-	return filepath.Join(c.RootRepoPath, "dists", distro, "main/binary-"+arch)
-}
-
-func (c Conf) PoolPath(distro, arch string) string {
-	return filepath.Join(c.RootRepoPath, "pool/main", distro, arch)
-}
-
-func (c Conf) PoolPackagePath(distro, arch, name string) string {
-	name = packageName(name)
-	return filepath.Join(c.RootRepoPath, "pool/main", distro, arch, name[0:1], name)
-}
-
 func packageName(name string) string {
 	if index := strings.Index(name, "_"); index != -1 {
 		name = name[:index]
@@ -63,9 +32,9 @@ type DeleteObj struct {
 
 var (
 	mutex        sync.RWMutex
-	configFile   = flag.String("c", "conf.json", "config file location")
+	configFile   = flag.String("c", "conf.ini", "config file location")
 	flagShowVersion = flag.Bool("version", false, "Show dnsconfig version")
-	parsedConfig = Conf{}
+	conf = Conf{}
 	pgpEntity *openpgp.Entity
 )
 
@@ -83,45 +52,53 @@ func Start() {
 		log.Fatalln("unable to read config file, exiting...")
 	}
 
-	if err := json.Unmarshal(file, &parsedConfig); err != nil {
+	if err := ini.MapTo(&conf, file); err != nil {
-		log.Fatalln("unable to marshal config file, exiting...")
+		log.Fatalln("unable to marshal config file, exiting...", err)
 	}
 
-	if err := createDirs(parsedConfig); err != nil {
+	if err := createDirs(conf); err != nil {
 		log.Println(err)
 		log.Fatalln("error creating directory structure, exiting")
 	}
 
-	if err := setupPgp(parsedConfig); err != nil {
+	if err := setupPgp(conf); err != nil {
 		log.Println(err)
 		log.Fatalln("error loading pgp key, exiting")
 	}
 
 	log.Println("Indexing packages...")
 
-	for _, dist := range parsedConfig.DistroNames {
+	for _, dist := range conf.Repo.DistroNames() {
-		distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)}
+		if err := loadCache(dist); err != nil {
+			log.Println("Unable to load cached data for", dist, "- reindexing")
+			distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)}
 
-		go scanInitialPackages(parsedConfig, distro)
+			go scanInitialPackages(conf, distro)
 
-		distros[dist] = distro
+			distros[dist] = distro
+		}
 	}
 
-	http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(parsedConfig.RootRepoPath))))
+	mux := http.NewServeMux()
-	http.Handle("/upload", uploadHandler(parsedConfig))
-	http.Handle("/delete", deleteHandler(parsedConfig))
 
-	if parsedConfig.EnableSSL {
+	mux.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(conf.Repo.Root))))
+	mux.HandleFunc("/rescan", rescanHandler)
+	mux.HandleFunc("/upload", uploadHandler)
+	mux.HandleFunc("/delete", deleteHandler)
+
+	bind := fmt.Sprintf(":%d", conf.Http.Port)
+
+	if conf.Http.SSL {
 		log.Println("running with SSL enabled")
-		log.Fatalln(http.ListenAndServeTLS(":"+parsedConfig.ListenPort, parsedConfig.SSLCert, parsedConfig.SSLKey, nil))
+		log.Fatalln(http.ListenAndServeTLS(bind, conf.Http.SSLCert, conf.Http.SSLKey, mux))
 	} else {
 		log.Println("running without SSL enabled")
-		log.Fatalln(http.ListenAndServe(":"+parsedConfig.ListenPort, nil))
+		log.Fatalln(http.ListenAndServe(bind, mux))
 	}
 }
 
 func scanInitialPackages(config Conf, dist *Distro) {
-	for _, arch := range config.SupportArch {
+	for _, arch := range config.Repo.ArchitectureNames() {
 		files, err := buildPackageList(config, dist.Name, arch)
 
 		if err != nil {
@@ -134,14 +111,16 @@ func scanInitialPackages(config Conf, dist *Distro) {
 		createPackagesCached(config, dist.Name, arch, files)
 		createRelease(config, dist.Name, arch)
 	}
+
+	saveCache(dist)
 }
 
 func setupPgp(config Conf) error {
-	if config.PGPSecretKey == "" {
+	if config.PGP.Key == "" {
 		return nil
 	}
 
-	secretKey, err := os.Open(config.PGPSecretKey)
+	secretKey, err := os.Open(config.PGP.Key)
 	if err != nil {
 		return fmt.Errorf("failed to open private key ring file: %s", err)
 	}
@@ -159,7 +138,7 @@ func setupPgp(config Conf) error {
 
 	pgpEntity = entitylist[0]
 
-	passphrase := []byte(config.PGPPassphrase)
+	passphrase := []byte(config.PGP.Passphrase)
 
 	if err := pgpEntity.PrivateKey.Decrypt(passphrase); err != nil {
 		return err
@@ -177,8 +156,8 @@ func setupPgp(config Conf) error {
 }
 
 func createDirs(config Conf) error {
-	for _, distro := range config.DistroNames {
+	for _, distro := range config.Repo.DistroNames() {
-		for _, arch := range config.SupportArch {
+		for _, arch := range config.Repo.ArchitectureNames() {
 			if _, err := os.Stat(config.ArchPath(distro, arch)); err != nil {
 				if os.IsNotExist(err) {
 					log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)