From 2ec6b2b8be1f6731c370904701a2739351779b36 Mon Sep 17 00:00:00 2001 From: Tyler Date: Wed, 13 Sep 2017 23:33:29 -0400 Subject: [PATCH] Improve directory handling, multi arch upload --- src/meow.tf/deb-simple/apt.go | 16 ++-- src/meow.tf/deb-simple/config.go | 14 ++-- src/meow.tf/deb-simple/http.go | 113 +++++++++++++++++++---------- src/meow.tf/deb-simple/packages.go | 28 +++---- src/meow.tf/deb-simple/server.go | 13 +++- 5 files changed, 118 insertions(+), 66 deletions(-) diff --git a/src/meow.tf/deb-simple/apt.go b/src/meow.tf/deb-simple/apt.go index b3ec0a9..4653b3d 100644 --- a/src/meow.tf/deb-simple/apt.go +++ b/src/meow.tf/deb-simple/apt.go @@ -4,7 +4,6 @@ import ( "os" "fmt" "io" - "path/filepath" "io/ioutil" "strings" "crypto/md5" @@ -14,11 +13,12 @@ import ( "golang.org/x/crypto/openpgp" "time" "meow.tf/deb-simple/deb/release" + "path" ) func createRelease(config Conf, distro string) error { - outfile, err := os.Create(filepath.Join(config.DistPath(distro), "Release")) + outfile, err := os.Create(path.Join(config.DistPath(distro), "Release")) if err != nil { return fmt.Errorf("failed to create Release: %s", err) @@ -34,7 +34,7 @@ func createRelease(config Conf, distro string) error { } for _, arch := range config.Repo.ArchitectureNames() { - absolutePath := filepath.Join(config.DistPath(distro), "main", "binary-" + arch) + absolutePath := path.Join(config.DistPath(distro), "main", "binary-" + arch) list, err := ioutil.ReadDir(absolutePath) @@ -42,12 +42,12 @@ func createRelease(config Conf, distro string) error { continue } - basePath := filepath.Join("main", "binary-" + arch) + basePath := path.Join("main", "binary-" + arch) for _, file := range list { - filePath := filepath.Join(absolutePath, file.Name()) + filePath := path.Join(absolutePath, file.Name()) - fileLocalPath := filepath.Join(basePath, file.Name()) + fileLocalPath := path.Join(basePath, file.Name()) fileLocalPath = strings.Replace(fileLocalPath, "\\", "/", -1) @@ -102,7 +102,7 @@ func createRelease(config Conf, distro string) error { func signRelease(config Conf, distro string) error { distPath := config.DistPath(distro) - f, err := os.Open(filepath.Join(distPath, "Release")) + f, err := os.Open(path.Join(distPath, "Release")) if err != nil { return fmt.Errorf("failed to read Release: %s", err) @@ -110,7 +110,7 @@ func signRelease(config Conf, distro string) error { defer f.Close() - gpgfile, err := os.Create(filepath.Join(distPath, "Release.gpg")) + gpgfile, err := os.Create(path.Join(distPath, "Release.gpg")) if err != nil { return fmt.Errorf("failed to create Release.gpg: %s", err) diff --git a/src/meow.tf/deb-simple/config.go b/src/meow.tf/deb-simple/config.go index 639e93d..75235e9 100644 --- a/src/meow.tf/deb-simple/config.go +++ b/src/meow.tf/deb-simple/config.go @@ -1,8 +1,8 @@ package simple import ( - "path/filepath" "strings" + "path" ) type Conf struct { @@ -31,20 +31,24 @@ type PGPConf struct { } func (c Conf) DistPath(distro string) string { - return filepath.Join(c.Repo.Root, "dists", distro) + return path.Join(c.Repo.Root, "dists", distro) } func (c Conf) ArchPath(distro, arch string) string { - return filepath.Join(c.Repo.Root, "dists", distro, "main/binary-"+arch) + return path.Join(c.Repo.Root, "dists", distro, "main/binary-"+arch) } func (c Conf) PoolPath(distro, arch string) string { - return filepath.Join(c.Repo.Root, "pool/main", distro, arch) + return path.Join(c.Repo.Root, "pool/main", distro, arch) } func (c Conf) PoolPackagePath(distro, arch, name string) string { + return path.Join(c.Repo.Root, c.RelativePoolPackagePath(distro, arch, name)) +} + +func (c Conf) RelativePoolPackagePath(distro, arch, name string) string { name = packageName(name) - return filepath.Join(c.Repo.Root, "pool/main", distro, arch, name[0:1], name) + return path.Join(c.Repo.Root, "pool/main", distro, arch, name[0:1], name) } func (c RepoConf) DistroNames() []string { diff --git a/src/meow.tf/deb-simple/http.go b/src/meow.tf/deb-simple/http.go index b05edc3..c74139a 100644 --- a/src/meow.tf/deb-simple/http.go +++ b/src/meow.tf/deb-simple/http.go @@ -4,14 +4,18 @@ import ( "net/http" "io" "os" - "path/filepath" "log" "encoding/json" "fmt" "github.com/blang/semver" + "strings" + "path" ) func rescanHandler(w http.ResponseWriter, r *http.Request) { + mutex.Lock() + defer mutex.Unlock() + distroName := r.URL.Query().Get("distro") if distroName == "" { @@ -27,7 +31,12 @@ func rescanHandler(w http.ResponseWriter, r *http.Request) { scanInitialPackages(conf, distro) + saveCache(distro) + distros[distroName] = distro + + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) } func uploadHandler(w http.ResponseWriter, r *http.Request) { @@ -36,10 +45,10 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { return } - archType := r.URL.Query().Get("arch") + queryArchType := r.URL.Query().Get("arch") - if archType == "" { - archType = "all" + if queryArchType == "" { + queryArchType = "all" } distroName := r.URL.Query().Get("distro") @@ -59,7 +68,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { forceStr := r.URL.Query().Get("force") - if forceStr != "" && forceStr == "true" { + if forceStr == "true" { force = true } @@ -80,20 +89,14 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { return } - packages, exists := distro.Architectures[archType] - - if !exists { - httpErrorf(w, "invalid arch: %s", archType) - mutex.RUnlock() - return - } - mutex.RUnlock() // Lock to prevent concurrent modification mutex.Lock() defer mutex.Unlock() + modifiedArches := make(map[string]bool) + for { part, err := reader.NextPart() @@ -101,10 +104,51 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { break } - if part.FileName() == "" { + if part.FileName() == "" || !strings.HasSuffix(part.FileName(), ".deb"){ continue } - newPath := conf.PoolPackagePath(distroName, archType, part.FileName()) + + tempFile := path.Join(os.TempDir(), "temp_" + part.FileName()) + + dst, err := os.Create(tempFile) + + if err != nil { + httpErrorf(w, "error creating deb file: %s", err) + return + } + + if _, err := io.Copy(dst, part); err != nil { + dst.Close() + + httpErrorf(w, "error writing deb file: %s", err) + return + } + + dst.Close() + + // Get package name, if it already exists remove the old file. + f, err := newPackageFile(tempFile, part.FileName()) + + if err != nil { + httpErrorf(w, "error loading package info: %s", err) + } + + archType := f.Info.Architecture + + if archType == "" { + archType = queryArchType + } + + // Get current packages + packages, exists := distro.Architectures[archType] + + if !exists { + httpErrorf(w, "invalid arch: %s", archType) + return + } + + // New path based off package data + newPath := conf.PoolPackagePath(distroName, archType, f.Info.Package) if _, err := os.Stat(newPath); err != nil { if os.IsNotExist(err) { @@ -115,27 +159,13 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { } } - dst, err := os.Create(filepath.Join(newPath, part.FileName())) + f.Path = path.Join(conf.RelativePoolPackagePath(distroName, archType, f.Info.Package), part.FileName()) - if err != nil { - httpErrorf(w, "error creating deb file: %s", err) + if err := os.Rename(tempFile, path.Join(newPath, part.FileName())); err != nil { + httpErrorf(w, "error moving temporary file: %s", err) return } - defer dst.Close() - - if _, err := io.Copy(dst, part); err != nil { - httpErrorf(w, "error writing deb file: %s", err) - return - } - - // Get package name, if it already exists remove the old file. - f, err := newPackageFile(newPath, part.FileName()) - - if err != nil { - httpErrorf(w, "error loading package info: %s", err) - } - if p, exists := packages[f.Info.Package]; exists { v1, err1 := semver.Parse(p.Info.Version) v2, err2 := semver.Parse(f.Info.Version) @@ -149,7 +179,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { // Archive old file log.Println("Replacing", p.Name, "with", f.Name) - if err := os.Remove(p.Path); err != nil { + if err := os.Remove(path.Join(conf.Repo.Root, p.Path)); err != nil { httpErrorf(w, "Unable to remove old package: %s", err) return } @@ -162,9 +192,11 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { // Recreate the package index and release file. - if err := createPackagesCached(conf, distroName, archType, packages); err != nil { - httpErrorf(w, "error creating package: %s", err) - return + for archType, _ := range modifiedArches { + if err := createPackagesCached(conf, distroName, archType, distro.Architectures[archType]); err != nil { + httpErrorf(w, "error creating package: %s", err) + return + } } err = createRelease(conf, distroName) @@ -174,6 +206,13 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { return } + err = saveCache(distro) + + if err != nil { + httpErrorf(w, "error updating cache: %s", err) + return + } + w.WriteHeader(http.StatusCreated) } @@ -216,7 +255,7 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) { return } - debPath := filepath.Join(conf.ArchPath(req.DistroName, req.Arch), req.Filename) + debPath := path.Join(conf.ArchPath(req.DistroName, req.Arch), req.Filename) if err := os.Remove(debPath); err != nil { httpErrorf(w, "failed to delete: %s", err) diff --git a/src/meow.tf/deb-simple/packages.go b/src/meow.tf/deb-simple/packages.go index 16470c2..00ee52b 100644 --- a/src/meow.tf/deb-simple/packages.go +++ b/src/meow.tf/deb-simple/packages.go @@ -3,7 +3,6 @@ package simple import ( "regexp" "fmt" - "path/filepath" "strings" "os" "bytes" @@ -18,6 +17,7 @@ import ( "github.com/blang/semver" "encoding/json" "meow.tf/deb-simple/deb/archive" + "path" ) var ( @@ -68,7 +68,7 @@ var ( ) func loadCache(dist string) error { - f, err := os.Open(filepath.Join(conf.DistPath(dist), "dist.json")) + f, err := os.Open(path.Join(conf.DistPath(dist), "dist.json")) if err != nil { return err @@ -88,7 +88,7 @@ func loadCache(dist string) error { } func saveCache(dist *Distro) error { - f, err := os.Create(filepath.Join(conf.DistPath(dist.Name), "dist.json")) + f, err := os.Create(path.Join(conf.DistPath(dist.Name), "dist.json")) if err != nil { return err @@ -113,8 +113,8 @@ func buildPackageList(config Conf, distro, arch string) (map[string]*PackageFile return m, nil } -func scanRecursive(path string, m map[string]*PackageFile) error { - dirList, err := ioutil.ReadDir(path) +func scanRecursive(base string, m map[string]*PackageFile) error { + dirList, err := ioutil.ReadDir(base) if err != nil { return err @@ -122,7 +122,7 @@ func scanRecursive(path string, m map[string]*PackageFile) error { for _, file := range dirList { if file.IsDir() { - if err := scanRecursive(filepath.Join(path, file.Name()), m); err != nil { + if err := scanRecursive(path.Join(base, file.Name()), m); err != nil { return err } continue @@ -132,7 +132,7 @@ func scanRecursive(path string, m map[string]*PackageFile) error { continue } - p, err := newPackageFile(path, file.Name()) + p, err := newPackageFile(base, file.Name()) if err != nil { return err @@ -154,13 +154,13 @@ func scanRecursive(path string, m map[string]*PackageFile) error { return nil } -func newPackageFile(path, name string) (*PackageFile, error) { +func newPackageFile(base, name string) (*PackageFile, error) { p := &PackageFile{ Name: name, - Path: filepath.Join(path, name), + Path: path.Join(stripPrefix(base, conf.Repo.Root), name), } - debPath := filepath.Join(path, name) + debPath := path.Join(base, name) var err error @@ -202,13 +202,13 @@ func newPackageFile(path, name string) (*PackageFile, error) { } func createPackagesCached(config Conf, distro, arch string, packages map[string]*PackageFile) error { - stdFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages")) + stdFile, err := os.Create(path.Join(config.ArchPath(distro, arch), "Packages")) if err != nil { return fmt.Errorf("failed to create packages: %s", err) } defer stdFile.Close() - gzipFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages.gz")) + gzipFile, err := os.Create(path.Join(config.ArchPath(distro, arch), "Packages.gz")) if err != nil { return fmt.Errorf("failed to create packages.gz: %s", err) } @@ -227,9 +227,9 @@ func createPackagesCached(config Conf, distro, arch string, packages map[string] packBuf.WriteString(p.ControlData) - dir := filepath.Join("pool/main", distro, arch, p.Info.Package[0:1], p.Info.Package, p.Name) + dir := path.Join("pool/main", distro, arch, p.Info.Package[0:1], p.Info.Package, p.Name) - fmt.Fprintf(&packBuf, "Filename: %s\n", strings.Replace(dir, "\\", "/", -1)) + fmt.Fprintf(&packBuf, "Filename: %s\n", dir) fmt.Fprintf(&packBuf, "Size: %d\n", p.Size) fmt.Fprintf(&packBuf, "MD5sum: %s\n", p.MD5Hash) diff --git a/src/meow.tf/deb-simple/server.go b/src/meow.tf/deb-simple/server.go index cd68dbb..65f00ab 100644 --- a/src/meow.tf/deb-simple/server.go +++ b/src/meow.tf/deb-simple/server.go @@ -15,7 +15,7 @@ import ( "github.com/go-ini/ini" ) -var VERSION string = "1.2.0" +var VERSION string = "1.3.0" func packageName(name string) string { if index := strings.Index(name, "_"); index != -1 { @@ -109,9 +109,11 @@ func scanInitialPackages(config Conf, dist *Distro) { log.Println("Generating packages file for", dist.Name, arch) createPackagesCached(config, dist.Name, arch, files) - createRelease(config, dist.Name) } + log.Println("Generating Release for", dist.Name) + createRelease(config, dist.Name) + saveCache(dist) } @@ -182,3 +184,10 @@ func createDirs(config Conf) error { } return nil } + +func stripPrefix(str, prefix string) string { + if strings.Index(str, prefix) == 0 { + return strings.TrimLeft(str[len(prefix):], "/") + } + return str +} \ No newline at end of file