Improve directory handling, multi arch upload

src/meow.tf/deb-simple/apt.go

@@ -4,7 +4,6 @@ import (
 	"os"
 	"fmt"
 	"io"
-	"path/filepath"
 	"io/ioutil"
 	"strings"
 	"crypto/md5"
@@ -14,11 +13,12 @@ import (
 	"golang.org/x/crypto/openpgp"
 	"time"
 	"meow.tf/deb-simple/deb/release"
+	"path"
 )
 
 
 func createRelease(config Conf, distro string) error {
-	outfile, err := os.Create(filepath.Join(config.DistPath(distro), "Release"))
+	outfile, err := os.Create(path.Join(config.DistPath(distro), "Release"))
 
 	if err != nil {
 		return fmt.Errorf("failed to create Release: %s", err)
@@ -34,7 +34,7 @@ func createRelease(config Conf, distro string) error {
 	}
 
 	for _, arch := range config.Repo.ArchitectureNames() {
-		absolutePath := filepath.Join(config.DistPath(distro), "main", "binary-" + arch)
+		absolutePath := path.Join(config.DistPath(distro), "main", "binary-" + arch)
 
 		list, err := ioutil.ReadDir(absolutePath)
 
@@ -42,12 +42,12 @@ func createRelease(config Conf, distro string) error {
 			continue
 		}
 
-		basePath := filepath.Join("main", "binary-" + arch)
+		basePath := path.Join("main", "binary-" + arch)
 
 		for _, file := range list {
-			filePath := filepath.Join(absolutePath, file.Name())
+			filePath := path.Join(absolutePath, file.Name())
 
-			fileLocalPath := filepath.Join(basePath, file.Name())
+			fileLocalPath := path.Join(basePath, file.Name())
 
 			fileLocalPath = strings.Replace(fileLocalPath, "\\", "/", -1)
 
@@ -102,7 +102,7 @@ func createRelease(config Conf, distro string) error {
 func signRelease(config Conf, distro string) error {
 	distPath := config.DistPath(distro)
 
-	f, err := os.Open(filepath.Join(distPath, "Release"))
+	f, err := os.Open(path.Join(distPath, "Release"))
 
 	if err != nil {
 		return fmt.Errorf("failed to read Release: %s", err)
@@ -110,7 +110,7 @@ func signRelease(config Conf, distro string) error {
 
 	defer f.Close()
 
-	gpgfile, err := os.Create(filepath.Join(distPath, "Release.gpg"))
+	gpgfile, err := os.Create(path.Join(distPath, "Release.gpg"))
 
 	if err != nil {
 		return fmt.Errorf("failed to create Release.gpg: %s", err)

src/meow.tf/deb-simple/config.go

@@ -1,8 +1,8 @@
 package simple
 
 import (
-	"path/filepath"
 	"strings"
+	"path"
 )
 
 type Conf struct {
@@ -31,20 +31,24 @@ type PGPConf struct {
 }
 
 func (c Conf) DistPath(distro string) string {
-	return filepath.Join(c.Repo.Root, "dists", distro)
+	return path.Join(c.Repo.Root, "dists", distro)
 }
 
 func (c Conf) ArchPath(distro, arch string) string {
-	return filepath.Join(c.Repo.Root, "dists", distro, "main/binary-"+arch)
+	return path.Join(c.Repo.Root, "dists", distro, "main/binary-"+arch)
 }
 
 func (c Conf) PoolPath(distro, arch string) string {
-	return filepath.Join(c.Repo.Root, "pool/main", distro, arch)
+	return path.Join(c.Repo.Root, "pool/main", distro, arch)
 }
 
 func (c Conf) PoolPackagePath(distro, arch, name string) string {
+	return path.Join(c.Repo.Root, c.RelativePoolPackagePath(distro, arch, name))
+}
+
+func (c Conf) RelativePoolPackagePath(distro, arch, name string) string {
 	name = packageName(name)
-	return filepath.Join(c.Repo.Root, "pool/main", distro, arch, name[0:1], name)
+	return path.Join(c.Repo.Root, "pool/main", distro, arch, name[0:1], name)
 }
 
 func (c RepoConf) DistroNames() []string {

src/meow.tf/deb-simple/http.go

@@ -4,14 +4,18 @@ import (
 	"net/http"
 	"io"
 	"os"
-	"path/filepath"
 	"log"
 	"encoding/json"
 	"fmt"
 	"github.com/blang/semver"
+	"strings"
+	"path"
 )
 
 func rescanHandler(w http.ResponseWriter, r *http.Request) {
+	mutex.Lock()
+	defer mutex.Unlock()
+
 	distroName := r.URL.Query().Get("distro")
 
 	if distroName == "" {
@@ -27,7 +31,12 @@ func rescanHandler(w http.ResponseWriter, r *http.Request) {
 
 	scanInitialPackages(conf, distro)
 
+	saveCache(distro)
+
 	distros[distroName] = distro
+
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("ok"))
 }
 
 func uploadHandler(w http.ResponseWriter, r *http.Request) {
@@ -36,10 +45,10 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	archType := r.URL.Query().Get("arch")
+	queryArchType := r.URL.Query().Get("arch")
 
-	if archType == "" {
+	if queryArchType == "" {
-		archType = "all"
+		queryArchType = "all"
 	}
 
 	distroName := r.URL.Query().Get("distro")
@@ -59,7 +68,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 
 	forceStr := r.URL.Query().Get("force")
 
-	if forceStr != "" && forceStr == "true" {
+	if forceStr == "true" {
 		force = true
 	}
 
@@ -80,20 +89,14 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	packages, exists := distro.Architectures[archType]
-
-	if !exists {
-		httpErrorf(w, "invalid arch: %s", archType)
-		mutex.RUnlock()
-		return
-	}
-
 	mutex.RUnlock()
 
 	// Lock to prevent concurrent modification
 	mutex.Lock()
 	defer mutex.Unlock()
 
+	modifiedArches := make(map[string]bool)
+
 	for {
 		part, err := reader.NextPart()
 
@@ -101,10 +104,51 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 			break
 		}
 
-		if part.FileName() == "" {
+		if part.FileName() == "" || !strings.HasSuffix(part.FileName(), ".deb"){
 			continue
 		}
-		newPath := conf.PoolPackagePath(distroName, archType, part.FileName())
+
+		tempFile := path.Join(os.TempDir(), "temp_" + part.FileName())
+
+		dst, err := os.Create(tempFile)
+
+		if err != nil {
+			httpErrorf(w, "error creating deb file: %s", err)
+			return
+		}
+
+		if _, err := io.Copy(dst, part); err != nil {
+			dst.Close()
+
+			httpErrorf(w, "error writing deb file: %s", err)
+			return
+		}
+
+		dst.Close()
+
+		// Get package name, if it already exists remove the old file.
+		f, err := newPackageFile(tempFile, part.FileName())
+
+		if err != nil {
+			httpErrorf(w, "error loading package info: %s", err)
+		}
+
+		archType := f.Info.Architecture
+
+		if archType == "" {
+			archType = queryArchType
+		}
+
+		// Get current packages
+		packages, exists := distro.Architectures[archType]
+
+		if !exists {
+			httpErrorf(w, "invalid arch: %s", archType)
+			return
+		}
+
+		// New path based off package data
+		newPath := conf.PoolPackagePath(distroName, archType, f.Info.Package)
 
 		if _, err := os.Stat(newPath); err != nil {
 			if os.IsNotExist(err) {
@@ -115,27 +159,13 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-		dst, err := os.Create(filepath.Join(newPath, part.FileName()))
+		f.Path = path.Join(conf.RelativePoolPackagePath(distroName, archType, f.Info.Package), part.FileName())
 
-		if err != nil {
+		if err := os.Rename(tempFile, path.Join(newPath, part.FileName())); err != nil {
-			httpErrorf(w, "error creating deb file: %s", err)
+			httpErrorf(w, "error moving temporary file: %s", err)
 			return
 		}
 
-		defer dst.Close()
-
-		if _, err := io.Copy(dst, part); err != nil {
-			httpErrorf(w, "error writing deb file: %s", err)
-			return
-		}
-
-		// Get package name, if it already exists remove the old file.
-		f, err := newPackageFile(newPath, part.FileName())
-
-		if err != nil {
-			httpErrorf(w, "error loading package info: %s", err)
-		}
-
 		if p, exists := packages[f.Info.Package]; exists {
 			v1, err1 := semver.Parse(p.Info.Version)
 			v2, err2 := semver.Parse(f.Info.Version)
@@ -149,7 +179,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 			// Archive old file
 			log.Println("Replacing", p.Name, "with", f.Name)
 
-			if err := os.Remove(p.Path); err != nil {
+			if err := os.Remove(path.Join(conf.Repo.Root, p.Path)); err != nil {
 				httpErrorf(w, "Unable to remove old package: %s", err)
 				return
 			}
@@ -162,9 +192,11 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Recreate the package index and release file.
 
-	if err := createPackagesCached(conf, distroName, archType, packages); err != nil {
+	for archType, _ := range modifiedArches {
-		httpErrorf(w, "error creating package: %s", err)
+		if err := createPackagesCached(conf, distroName, archType, distro.Architectures[archType]); err != nil {
-		return
+			httpErrorf(w, "error creating package: %s", err)
+			return
+		}
 	}
 
 	err = createRelease(conf, distroName)
@@ -174,6 +206,13 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	err = saveCache(distro)
+
+	if err != nil {
+		httpErrorf(w, "error updating cache: %s", err)
+		return
+	}
+
 	w.WriteHeader(http.StatusCreated)
 }
 
@@ -216,7 +255,7 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	debPath := filepath.Join(conf.ArchPath(req.DistroName, req.Arch), req.Filename)
+	debPath := path.Join(conf.ArchPath(req.DistroName, req.Arch), req.Filename)
 
 	if err := os.Remove(debPath); err != nil {
 		httpErrorf(w, "failed to delete: %s", err)

src/meow.tf/deb-simple/packages.go

@@ -3,7 +3,6 @@ package simple
 import (
 	"regexp"
 	"fmt"
-	"path/filepath"
 	"strings"
 	"os"
 	"bytes"
@@ -18,6 +17,7 @@ import (
 	"github.com/blang/semver"
 	"encoding/json"
 	"meow.tf/deb-simple/deb/archive"
+	"path"
 )
 
 var (
@@ -68,7 +68,7 @@ var (
 )
 
 func loadCache(dist string) error {
-	f, err := os.Open(filepath.Join(conf.DistPath(dist), "dist.json"))
+	f, err := os.Open(path.Join(conf.DistPath(dist), "dist.json"))
 
 	if err != nil {
 		return err
@@ -88,7 +88,7 @@ func loadCache(dist string) error {
 }
 
 func saveCache(dist *Distro) error {
-	f, err := os.Create(filepath.Join(conf.DistPath(dist.Name), "dist.json"))
+	f, err := os.Create(path.Join(conf.DistPath(dist.Name), "dist.json"))
 
 	if err != nil {
 		return err
@@ -113,8 +113,8 @@ func buildPackageList(config Conf, distro, arch string) (map[string]*PackageFile
 	return m, nil
 }
 
-func scanRecursive(path string, m map[string]*PackageFile) error {
+func scanRecursive(base string, m map[string]*PackageFile) error {
-	dirList, err := ioutil.ReadDir(path)
+	dirList, err := ioutil.ReadDir(base)
 
 	if err != nil {
 		return err
@@ -122,7 +122,7 @@ func scanRecursive(path string, m map[string]*PackageFile) error {
 
 	for _, file := range dirList {
 		if file.IsDir() {
-			if err := scanRecursive(filepath.Join(path, file.Name()), m); err != nil {
+			if err := scanRecursive(path.Join(base, file.Name()), m); err != nil {
 				return err
 			}
 			continue
@@ -132,7 +132,7 @@ func scanRecursive(path string, m map[string]*PackageFile) error {
 			continue
 		}
 
-		p, err := newPackageFile(path, file.Name())
+		p, err := newPackageFile(base, file.Name())
 
 		if err != nil {
 			return err
@@ -154,13 +154,13 @@ func scanRecursive(path string, m map[string]*PackageFile) error {
 	return nil
 }
 
-func newPackageFile(path, name string) (*PackageFile, error) {
+func newPackageFile(base, name string) (*PackageFile, error) {
 	p := &PackageFile{
 		Name: name,
-		Path: filepath.Join(path, name),
+		Path: path.Join(stripPrefix(base, conf.Repo.Root), name),
 	}
 
-	debPath := filepath.Join(path, name)
+	debPath := path.Join(base, name)
 
 	var err error
 
@@ -202,13 +202,13 @@ func newPackageFile(path, name string) (*PackageFile, error) {
 }
 
 func createPackagesCached(config Conf, distro, arch string, packages map[string]*PackageFile) error {
-	stdFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages"))
+	stdFile, err := os.Create(path.Join(config.ArchPath(distro, arch), "Packages"))
 	if err != nil {
 		return fmt.Errorf("failed to create packages: %s", err)
 	}
 	defer stdFile.Close()
 
-	gzipFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages.gz"))
+	gzipFile, err := os.Create(path.Join(config.ArchPath(distro, arch), "Packages.gz"))
 	if err != nil {
 		return fmt.Errorf("failed to create packages.gz: %s", err)
 	}
@@ -227,9 +227,9 @@ func createPackagesCached(config Conf, distro, arch string, packages map[string]
 
 		packBuf.WriteString(p.ControlData)
 
-		dir := filepath.Join("pool/main", distro, arch, p.Info.Package[0:1], p.Info.Package, p.Name)
+		dir := path.Join("pool/main", distro, arch, p.Info.Package[0:1], p.Info.Package, p.Name)
 
-		fmt.Fprintf(&packBuf, "Filename: %s\n", strings.Replace(dir, "\\", "/", -1))
+		fmt.Fprintf(&packBuf, "Filename: %s\n", dir)
 		fmt.Fprintf(&packBuf, "Size: %d\n", p.Size)
 
 		fmt.Fprintf(&packBuf, "MD5sum: %s\n", p.MD5Hash)

src/meow.tf/deb-simple/server.go

@@ -15,7 +15,7 @@ import (
 	"github.com/go-ini/ini"
 )
 
-var VERSION string = "1.2.0"
+var VERSION string = "1.3.0"
 
 func packageName(name string) string {
 	if index := strings.Index(name, "_"); index != -1 {
@@ -109,9 +109,11 @@ func scanInitialPackages(config Conf, dist *Distro) {
 
 		log.Println("Generating packages file for", dist.Name, arch)
 		createPackagesCached(config, dist.Name, arch, files)
-		createRelease(config, dist.Name)
 	}
 
+	log.Println("Generating Release for", dist.Name)
+	createRelease(config, dist.Name)
+
 	saveCache(dist)
 }
 
@@ -182,3 +184,10 @@ func createDirs(config Conf) error {
 	}
 	return nil
 }
+
+func stripPrefix(str, prefix string) string {
+	if strings.Index(str, prefix) == 0 {
+		return strings.TrimLeft(str[len(prefix):], "/")
+	}
+	return str
+}