Fix tmpfs issues, add swapping of options, etc

server.go

@@ -9,11 +9,11 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	s3 "github.com/fclairamb/afero-s3"
 	"github.com/go-ini/ini"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 	"golang.org/x/crypto/openpgp"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"os"
 	"path"
@@ -41,15 +41,20 @@ var (
 	mutex        sync.RWMutex
 	configFile   = flag.String("c", "conf.ini", "config file location")
 	flagShowVersion = flag.Bool("version", false, "Show deb-simple version")
+	flagDebug = flag.Bool("debug", false, "debug output")
 	conf = Conf{}
 	pgpEntity *openpgp.Entity
 	fs afero.Fs
-	baseTempDir string
+	tmpFs afero.Fs
 )
 
 func main() {
 	flag.Parse()
 
+	if *flagDebug {
+		log.SetLevel(log.DebugLevel)
+	}
+
 	if *flagShowVersion {
 		fmt.Printf("deb-simple %s (%s)\n", VERSION, runtime.Version())
 		os.Exit(0)
@@ -58,23 +63,14 @@ func main() {
 	file, err := ioutil.ReadFile(*configFile)
 
 	if err != nil {
-		log.Fatalln("unable to read config file, exiting...")
+		log.WithError(err).Fatalln("unable to read config file, exiting...")
 	}
 
 	if err := ini.MapTo(&conf, file); err != nil {
-		log.Fatalln("unable to marshal config file, exiting...", err)
-	}
-
-	baseTempDir = path.Join(os.TempDir(), "deb-simple")
-
-	if _, err := os.Stat(baseTempDir); err != nil {
-		if os.IsNotExist(err) {
-			if err := os.MkdirAll(baseTempDir, 0755); err != nil {
-				return
-			}
-		}
+		log.WithError(err).Fatalln("unable to marshal config file, exiting...", err)
 	}
 
+	osFs := afero.NewOsFs()
 
 	switch conf.Fs.Driver {
 	case "s3":
@@ -97,24 +93,41 @@ func main() {
 	case "local":
 		fallthrough
 	default:
-		fs = afero.NewBasePathFs(afero.NewOsFs(), conf.Repo.Root)
+		fs = afero.NewBasePathFs(osFs, conf.Repo.Root)
+	}
+
+	switch conf.Fs.TmpDriver {
+	case "memory":
+		tmpFs = afero.NewMemMapFs()
+	case "local":
+		fallthrough
+	default:
+		baseTempDir := path.Join(os.TempDir(), "deb-simple")
+
+		if _, err := os.Stat(baseTempDir); err != nil {
+			if os.IsNotExist(err) {
+				if err := os.MkdirAll(baseTempDir, 0755); err != nil {
+					log.Fatalln("Unable to create temp dir:", err)
+				}
+			}
+		}
+
+		tmpFs = afero.NewBasePathFs(osFs, baseTempDir)
 	}
 
 	if err := createDirs(conf); err != nil {
-		log.Println(err)
-		log.Fatalln("error creating directory structure, exiting")
+		log.WithError(err).Fatalln("error creating directory structure, exiting")
 	}
 
 	if err := setupPgp(conf); err != nil {
-		log.Println(err)
-		log.Fatalln("error loading pgp key, exiting")
+		log.WithError(err).Fatalln("error loading pgp key, exiting")
 	}
 
-	log.Println("Indexing packages...")
+	log.Info("Indexing packages...")
 
 	for _, dist := range conf.Repo.DistroNames() {
 		if err := loadCache(dist); err != nil {
-			log.Println("Unable to load cached data for", dist, "- reindexing")
+			log.Info("Unable to load cached data for", dist, "- reindexing")
 			distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)}
 
 			go scanInitialPackages(conf, distro)
@@ -135,10 +148,10 @@ func main() {
 	bind := fmt.Sprintf(":%d", conf.Http.Port)
 
 	if conf.Http.SSL {
-		log.Println("running with SSL enabled")
+		log.Info("running with SSL enabled")
 		log.Fatalln(http.ListenAndServeTLS(bind, conf.Http.SSLCert, conf.Http.SSLKey, mux))
 	} else {
-		log.Println("running without SSL enabled")
+		log.Info("running without SSL enabled")
 		log.Fatalln(http.ListenAndServe(bind, mux))
 	}
 }
@@ -153,14 +166,27 @@ func scanInitialPackages(config Conf, dist *Distro) {
 
 		dist.Architectures[arch] = files
 
-		log.Println("Generating packages file for", dist.Name, arch)
-		createPackagesCached(config, dist.Name, arch, files)
+		log.Info("Generating packages file for", dist.Name, arch)
+
+		err = createPackagesCached(config, dist.Name, arch, files)
+
+		if err != nil {
+			log.WithError(err).Fatalln("Unable to generate package list")
+		}
 	}
 
-	log.Println("Generating Release for", dist.Name)
-	createRelease(config, dist.Name)
+	log.Info("Generating Release for", dist.Name)
+	err := createRelease(config, dist.Name)
 
-	saveCache(dist)
+	if err != nil {
+		log.WithError(err).Fatalln("Unable to generate release file")
+	}
+
+	err = saveCache(dist)
+
+	if err != nil {
+		log.WithError(err).Fatalln("Unable to save cache")
+	}
 }
 
 func setupPgp(config Conf) error {
@@ -208,7 +234,7 @@ func createDirs(config Conf) error {
 		for _, arch := range config.Repo.ArchitectureNames() {
 			if _, err := fs.Stat(config.ArchPath(distro, arch)); err != nil {
 				if os.IsNotExist(err) {
-					log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)
+					log.Debugf("Directory for %s (%s) does not exist, creating", distro, arch)
 					if err := fs.MkdirAll(config.ArchPath(distro, arch), 0755); err != nil {
 						return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err)
 					}
@@ -218,7 +244,7 @@ func createDirs(config Conf) error {
 			}
 			if _, err := fs.Stat(config.PoolPath(distro, arch)); err != nil {
 				if os.IsNotExist(err) {
-					log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)
+					log.Debugf("Directory for %s (%s) does not exist, creating", distro, arch)
 					if err := fs.MkdirAll(config.PoolPath(distro, arch), 0755); err != nil {
 						return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err)
 					}
@@ -231,15 +257,8 @@ func createDirs(config Conf) error {
 	return nil
 }
 
-func stripPrefix(str, prefix string) string {
-	if strings.Index(str, prefix) == 0 {
-		return strings.TrimLeft(str[len(prefix):], "/")
-	}
-	return str
-}
-
-func copyFile(oldPath, newPath string) error {
-	old, err := os.Open(oldPath)
+func copyFile(sourceFs afero.Fs, oldPath string, destFs afero.Fs, newPath string) error {
+	old, err := sourceFs.Open(oldPath)
 
 	if err != nil {
 		return err
@@ -247,7 +266,7 @@ func copyFile(oldPath, newPath string) error {
 
 	defer old.Close()
 
-	n, err := fs.Create(newPath)
+	n, err := destFs.Create(newPath)
 
 	if err != nil {
 		return err