Fix tmpfs issues, add swapping of options, etc

.drone.yml

@@ -2,38 +2,14 @@ kind: pipeline
 name: default
 
 steps:
-  - name: build-i386
+  - name: build
     image: golang:latest
     group: build
     commands:
      - mkdir -p /build/i386
      - GOOS=linux GOARCH=386 go build -o /build/i386/deb-simple
-    volumes:
+     - GOOS=linux GOARCH=amd64 go build -o /build/amd64/deb-simple
-      - name: build
-        path: /build
-  - name: build-amd64
-    image: golang:latest
-    group: build
-    commands:
-     - mkdir -p /build/amd64
-     - go build -o /build/amd64/deb-simple\
-    volumes:
-      - name: build
-        path: /build
-  - name: build-armv7
-    image: golang:latest
-    group: build
-    commands:
-      - mkdir -p /build/armv7
      - GOOS=linux GOARCH=arm GOARM=7 go build -o /build/armv7/deb-simple
-    volumes:
-      - name: build
-        path: /build
-  - name: build-arm64
-    image: golang:latest
-    group: build
-    commands:
-      - mkdir -p /build/arm64
      - GOOS=linux GOARCH=arm64 go build -o /build/arm64/deb-simple
     volumes:
       - name: build

config.go

@@ -14,6 +14,7 @@ type Conf struct {
 
 type FsConf struct {
 	Driver string `ini:"driver"`
+	TmpDriver string `ini:"tmpDriver"`
 	S3 struct {
 		Bucket string `ini:"bucket"`
 		Region string `ini:"region"`

deb/archive/archive.go

@@ -8,19 +8,11 @@ import (
 	"github.com/blakesmith/ar"
 	"io"
 	"log"
-	"os"
 	"strings"
 )
 
-func InspectPackage(filename string) (string, error) {
+func InspectPackage(r io.Reader) (string, error) {
-	f, err := os.Open(filename)
+	arReader := ar.NewReader(r)
-
-	if err != nil {
-		return "", fmt.Errorf("error opening package file %s: %s", filename, err)
-	}
-
-	arReader := ar.NewReader(f)
-	defer f.Close()
 
 	for {
 		header, err := arReader.Next()
@@ -37,6 +29,7 @@ func InspectPackage(filename string) (string, error) {
 			return InspectPackageControl(arReader)
 		}
 	}
+
 	return "", nil
 }
 

glide.lock

@@ -1,20 +0,0 @@
-hash: 1a6d5cdf757d6be4cab4e8b0d2224a38038fb086e135b15282ba85d02c54368a
-updated: 2017-09-14T00:26:28.2385236-04:00
-imports:
-- name: github.com/blakesmith/ar
-  version: 8bd4349a67f2533b078dbc524689d15dba0f4659
-- name: github.com/blang/semver
-  version: 2ee87856327ba09384cabd113bc6b5d174e9ec0f
-- name: github.com/go-ini/ini
-  version: 20b96f641a5ea98f2f8619ff4f3e061cff4833bd
-- name: golang.org/x/crypto
-  version: 7e9105388ebff089b3f99f0ef676ea55a6da3a7e
-  subpackages:
-  - cast5
-  - openpgp
-  - openpgp/armor
-  - openpgp/elgamal
-  - openpgp/errors
-  - openpgp/packet
-  - openpgp/s2k
-testImports: []

glide.yaml

@@ -1,10 +0,0 @@
-package: meow.tf/deb-simple
-import:
-- package: github.com/blakesmith/ar
-- package: golang.org/x/crypto
-  subpackages:
-  - openpgp
-- package: github.com/blang/semver
-  version: ^3.5.0
-- package: github.com/go-ini/ini
-  version: ^1.28.0

go.mod

@@ -3,10 +3,12 @@ module meow.tf/deb-simple
 go 1.14
 
 require (
+	github.com/aws/aws-sdk-go v1.31.3
 	github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/fclairamb/afero-s3 v0.1.0 // indirect
+	github.com/fclairamb/afero-s3 v0.1.0
 	github.com/go-ini/ini v1.28.2
+	github.com/sirupsen/logrus v1.6.0
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/afero v1.2.2
 	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2

go.sum

@@ -5,6 +5,7 @@ github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2/go.mod h1:PkYb9DJNAw
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fclairamb/afero-s3 v0.1.0 h1:WfPD2Px6AlcEBw1jUmJC295wpTXkfglYtzbSdTfaNzg=
 github.com/fclairamb/afero-s3 v0.1.0/go.mod h1:HHcYBsOLXjeqdnKQ61Rx43+vj+jGe9DjYTQbKhendwQ=
 github.com/go-ini/ini v1.28.2 h1:drmmYv7psRpoGZkPtPKKTB+ZFSnvmwCMfNj5o1nLh2Y=
@@ -16,8 +17,12 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
@@ -25,6 +30,7 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 golang.org/x/crypto v0.0.0-20170523101029-7e9105388ebf h1:Ak4rvIfvuVxAE1NgtCq1GvH0e5psMKmzowfoGdL8QBA=
 golang.org/x/crypto v0.0.0-20170523101029-7e9105388ebf/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -33,6 +39,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=

http.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/blang/semver"
+	log "github.com/sirupsen/logrus"
 	"io"
-	"log"
 	"net/http"
 	"os"
 	"path"
@@ -140,17 +140,23 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 		modifiedArches[archType] = true
 	}
 
-	log.Println("got lock, updating package list...")
-
 	// Recreate the package index and release file.
 
 	for archType, _ := range modifiedArches {
+		log.WithFields(log.Fields{
+			"arch": archType,
+		}).Debug("Updating package list...")
+
 		if err := createPackagesCached(conf, distroName, archType, distro.Architectures[archType]); err != nil {
 			httpErrorf(w, http.StatusInternalServerError, "error creating package: %s", err)
 			return
 		}
 	}
 
+	log.WithFields(log.Fields{
+		"distroName": distroName,
+	}).Debug("Updating release files...")
+
 	err = createRelease(conf, distroName)
 
 	if err != nil {
@@ -158,6 +164,8 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	log.Debug("Dumping cache...")
+
 	err = saveCache(distro)
 
 	if err != nil {
@@ -165,13 +173,13 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	log.Debug("Done.")
+
 	w.WriteHeader(http.StatusCreated)
 }
 
 func copyToTemp(fileName string, r io.Reader) (string, error) {
-	tempFile := path.Join(baseTempDir, fileName)
+	dst, err := tmpFs.Create(fileName)
-
-	dst, err := os.Create(tempFile)
 
 	if err != nil {
 		return "", fmt.Errorf("error creating deb file: %s", err)
@@ -194,7 +202,7 @@ func loadAndCheckPackage(distroName string, distro *Distro, fileName string, r i
 	}
 
 	// Get package name, if it already exists remove the old file.
-	f, err := newPackageFile(tempFile)
+	f, err := newPackageFile(tmpFs, tempFile)
 
 	if err != nil {
 		return "", fmt.Errorf("error loading package info: %s", err)
@@ -222,11 +230,11 @@ func loadAndCheckPackage(distroName string, distro *Distro, fileName string, r i
 
 	f.Path = path.Join(conf.RelativePoolPackagePath(distroName, archType, f.Info.Package), fileName)
 
-	if err := copyFile(tempFile, path.Join(newPath, fileName)); err != nil {
+	if err := copyFile(tmpFs, tempFile, fs, path.Join(newPath, fileName)); err != nil {
 		return "", fmt.Errorf("error copying temporary file: %s", err)
 	}
 
-	if err := fs.Remove(tempFile); err != nil {
+	if err := tmpFs.Remove(tempFile); err != nil {
 		return "", fmt.Errorf("unable to remove temporary file: %s", err)
 	}
 
@@ -242,10 +250,13 @@ func loadAndCheckPackage(distroName string, distro *Distro, fileName string, r i
 		}
 
 		// Archive old file
-		log.Println("Replacing", p.Name, "with", f.Name)
+		log.WithFields(log.Fields{
+			"existing": p.Path,
+			"new": f.Path,
+		}).Debug("Replacing existing file")
 
 		// If oldPath == newPath then we already overwrote it
-		if path.Base(p.Path) != path.Base(newPath) {
+		if path.Base(p.Path) != path.Base(f.Path) {
 			if err := fs.Remove(p.Path); err != nil && !os.IsNotExist(err) {
 				return "", fmt.Errorf("unable to remove old package: %s", err)
 			}

packages.go

@@ -130,7 +130,7 @@ func scanRecursive(base string, m map[string]*PackageFile) error {
 			continue
 		}
 
-		p, err := newPackageFile(path.Join(base, file.Name()))
+		p, err := newPackageFile(fs, path.Join(base, file.Name()))
 
 		if err != nil {
 			return err
@@ -152,27 +152,23 @@ func scanRecursive(base string, m map[string]*PackageFile) error {
 	return nil
 }
 
-func newPackageFile(filePath string) (*PackageFile, error) {
+func newPackageFile(sourceFs afero.Fs, filePath string) (*PackageFile, error) {
 	p := &PackageFile{
 		Name: path.Base(filePath),
-		Path: stripPrefix(filePath, conf.Repo.Root),
+		Path: filePath,
 	}
 
 	var err error
 
-	p.ControlData, err = archive.InspectPackage(filePath)
+	stat, err := sourceFs.Stat(filePath)
 
 	if err != nil {
 		return nil, err
 	}
 
-	p.Info = parsePackageData(p.ControlData)
-
-	if stat, err := fs.Stat(filePath); err == nil {
 	p.Size = stat.Size()
-	}
 
-	f, err := fs.Open(filePath)
+	f, err := sourceFs.Open(filePath)
 
 	if err != nil {
 		return nil, err
@@ -180,6 +176,18 @@ func newPackageFile(filePath string) (*PackageFile, error) {
 
 	defer f.Close()
 
+	p.ControlData, err = archive.InspectPackage(f)
+
+	p.Info = parsePackageData(p.ControlData)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = f.Seek(0, io.SeekStart); err != nil {
+		return nil, err
+	}
+
 	var (
 		md5hash    = md5.New()
 		sha1hash   = sha1.New()

server.go

@@ -9,11 +9,11 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	s3 "github.com/fclairamb/afero-s3"
 	"github.com/go-ini/ini"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 	"golang.org/x/crypto/openpgp"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"os"
 	"path"
@@ -41,15 +41,20 @@ var (
 	mutex        sync.RWMutex
 	configFile   = flag.String("c", "conf.ini", "config file location")
 	flagShowVersion = flag.Bool("version", false, "Show deb-simple version")
+	flagDebug = flag.Bool("debug", false, "debug output")
 	conf = Conf{}
 	pgpEntity *openpgp.Entity
 	fs afero.Fs
-	baseTempDir string
+	tmpFs afero.Fs
 )
 
 func main() {
 	flag.Parse()
 
+	if *flagDebug {
+		log.SetLevel(log.DebugLevel)
+	}
+
 	if *flagShowVersion {
 		fmt.Printf("deb-simple %s (%s)\n", VERSION, runtime.Version())
 		os.Exit(0)
@@ -58,23 +63,14 @@ func main() {
 	file, err := ioutil.ReadFile(*configFile)
 
 	if err != nil {
-		log.Fatalln("unable to read config file, exiting...")
+		log.WithError(err).Fatalln("unable to read config file, exiting...")
 	}
 
 	if err := ini.MapTo(&conf, file); err != nil {
-		log.Fatalln("unable to marshal config file, exiting...", err)
+		log.WithError(err).Fatalln("unable to marshal config file, exiting...", err)
-	}
-
-	baseTempDir = path.Join(os.TempDir(), "deb-simple")
-
-	if _, err := os.Stat(baseTempDir); err != nil {
-		if os.IsNotExist(err) {
-			if err := os.MkdirAll(baseTempDir, 0755); err != nil {
-				return
-			}
-		}
 	}
 
+	osFs := afero.NewOsFs()
 
 	switch conf.Fs.Driver {
 	case "s3":
@@ -97,24 +93,41 @@ func main() {
 	case "local":
 		fallthrough
 	default:
-		fs = afero.NewBasePathFs(afero.NewOsFs(), conf.Repo.Root)
+		fs = afero.NewBasePathFs(osFs, conf.Repo.Root)
+	}
+
+	switch conf.Fs.TmpDriver {
+	case "memory":
+		tmpFs = afero.NewMemMapFs()
+	case "local":
+		fallthrough
+	default:
+		baseTempDir := path.Join(os.TempDir(), "deb-simple")
+
+		if _, err := os.Stat(baseTempDir); err != nil {
+			if os.IsNotExist(err) {
+				if err := os.MkdirAll(baseTempDir, 0755); err != nil {
+					log.Fatalln("Unable to create temp dir:", err)
+				}
+			}
+		}
+
+		tmpFs = afero.NewBasePathFs(osFs, baseTempDir)
 	}
 
 	if err := createDirs(conf); err != nil {
-		log.Println(err)
+		log.WithError(err).Fatalln("error creating directory structure, exiting")
-		log.Fatalln("error creating directory structure, exiting")
 	}
 
 	if err := setupPgp(conf); err != nil {
-		log.Println(err)
+		log.WithError(err).Fatalln("error loading pgp key, exiting")
-		log.Fatalln("error loading pgp key, exiting")
 	}
 
-	log.Println("Indexing packages...")
+	log.Info("Indexing packages...")
 
 	for _, dist := range conf.Repo.DistroNames() {
 		if err := loadCache(dist); err != nil {
-			log.Println("Unable to load cached data for", dist, "- reindexing")
+			log.Info("Unable to load cached data for", dist, "- reindexing")
 			distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)}
 
 			go scanInitialPackages(conf, distro)
@@ -135,10 +148,10 @@ func main() {
 	bind := fmt.Sprintf(":%d", conf.Http.Port)
 
 	if conf.Http.SSL {
-		log.Println("running with SSL enabled")
+		log.Info("running with SSL enabled")
 		log.Fatalln(http.ListenAndServeTLS(bind, conf.Http.SSLCert, conf.Http.SSLKey, mux))
 	} else {
-		log.Println("running without SSL enabled")
+		log.Info("running without SSL enabled")
 		log.Fatalln(http.ListenAndServe(bind, mux))
 	}
 }
@@ -153,14 +166,27 @@ func scanInitialPackages(config Conf, dist *Distro) {
 
 		dist.Architectures[arch] = files
 
-		log.Println("Generating packages file for", dist.Name, arch)
+		log.Info("Generating packages file for", dist.Name, arch)
-		createPackagesCached(config, dist.Name, arch, files)
+
+		err = createPackagesCached(config, dist.Name, arch, files)
+
+		if err != nil {
+			log.WithError(err).Fatalln("Unable to generate package list")
+		}
 	}
 
-	log.Println("Generating Release for", dist.Name)
+	log.Info("Generating Release for", dist.Name)
-	createRelease(config, dist.Name)
+	err := createRelease(config, dist.Name)
 
-	saveCache(dist)
+	if err != nil {
+		log.WithError(err).Fatalln("Unable to generate release file")
+	}
+
+	err = saveCache(dist)
+
+	if err != nil {
+		log.WithError(err).Fatalln("Unable to save cache")
+	}
 }
 
 func setupPgp(config Conf) error {
@@ -208,7 +234,7 @@ func createDirs(config Conf) error {
 		for _, arch := range config.Repo.ArchitectureNames() {
 			if _, err := fs.Stat(config.ArchPath(distro, arch)); err != nil {
 				if os.IsNotExist(err) {
-					log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)
+					log.Debugf("Directory for %s (%s) does not exist, creating", distro, arch)
 					if err := fs.MkdirAll(config.ArchPath(distro, arch), 0755); err != nil {
 						return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err)
 					}
@@ -218,7 +244,7 @@ func createDirs(config Conf) error {
 			}
 			if _, err := fs.Stat(config.PoolPath(distro, arch)); err != nil {
 				if os.IsNotExist(err) {
-					log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)
+					log.Debugf("Directory for %s (%s) does not exist, creating", distro, arch)
 					if err := fs.MkdirAll(config.PoolPath(distro, arch), 0755); err != nil {
 						return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err)
 					}
@@ -231,15 +257,8 @@ func createDirs(config Conf) error {
 	return nil
 }
 
-func stripPrefix(str, prefix string) string {
+func copyFile(sourceFs afero.Fs, oldPath string, destFs afero.Fs, newPath string) error {
-	if strings.Index(str, prefix) == 0 {
+	old, err := sourceFs.Open(oldPath)
-		return strings.TrimLeft(str[len(prefix):], "/")
-	}
-	return str
-}
-
-func copyFile(oldPath, newPath string) error {
-	old, err := os.Open(oldPath)
 
 	if err != nil {
 		return err
@@ -247,7 +266,7 @@ func copyFile(oldPath, newPath string) error {
 
 	defer old.Close()
 
-	n, err := fs.Create(newPath)
+	n, err := destFs.Create(newPath)
 
 	if err != nil {
 		return err