tyler
/
deb-simple
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improve directory handling, multi arch upload

This commit is contained in:
Tyler 2017-09-13 23:33:29 -04:00
parent 0e001f1a3a
commit 2ec6b2b8be
5 changed files with 118 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/meow.tf/deb-simple/apt.go
View File

@ -4,7 +4,6 @@ import (
"os"
"fmt"
"io"
"path/filepath"
"io/ioutil"
"strings"
"crypto/md5"
@ -14,11 +13,12 @@ import (
"golang.org/x/crypto/openpgp"
"time"
"meow.tf/deb-simple/deb/release"
"path"
)
func createRelease(config Conf, distro string) error {
outfile, err := os.Create(filepath.Join(config.DistPath(distro), "Release"))
outfile, err := os.Create(path.Join(config.DistPath(distro), "Release"))
if err != nil {
return fmt.Errorf("failed to create Release: %s", err)
@ -34,7 +34,7 @@ func createRelease(config Conf, distro string) error {
}
for _, arch := range config.Repo.ArchitectureNames() {
absolutePath := filepath.Join(config.DistPath(distro), "main", "binary-" + arch)
absolutePath := path.Join(config.DistPath(distro), "main", "binary-" + arch)
list, err := ioutil.ReadDir(absolutePath)
@ -42,12 +42,12 @@ func createRelease(config Conf, distro string) error {
continue
}
basePath := filepath.Join("main", "binary-" + arch)
basePath := path.Join("main", "binary-" + arch)
for _, file := range list {
filePath := filepath.Join(absolutePath, file.Name())
filePath := path.Join(absolutePath, file.Name())
fileLocalPath := filepath.Join(basePath, file.Name())
fileLocalPath := path.Join(basePath, file.Name())
fileLocalPath = strings.Replace(fileLocalPath, "\\", "/", -1)
@ -102,7 +102,7 @@ func createRelease(config Conf, distro string) error {
func signRelease(config Conf, distro string) error {
distPath := config.DistPath(distro)
f, err := os.Open(filepath.Join(distPath, "Release"))
f, err := os.Open(path.Join(distPath, "Release"))
if err != nil {
return fmt.Errorf("failed to read Release: %s", err)
@ -110,7 +110,7 @@ func signRelease(config Conf, distro string) error {
defer f.Close()
gpgfile, err := os.Create(filepath.Join(distPath, "Release.gpg"))
gpgfile, err := os.Create(path.Join(distPath, "Release.gpg"))
if err != nil {
return fmt.Errorf("failed to create Release.gpg: %s", err)

14
src/meow.tf/deb-simple/config.go
View File

@ -1,8 +1,8 @@
package simple
import (
"path/filepath"
"strings"
"path"
)
type Conf struct {
@ -31,20 +31,24 @@ type PGPConf struct {
}
func (c Conf) DistPath(distro string) string {
return filepath.Join(c.Repo.Root, "dists", distro)
return path.Join(c.Repo.Root, "dists", distro)
}
func (c Conf) ArchPath(distro, arch string) string {
return filepath.Join(c.Repo.Root, "dists", distro, "main/binary-"+arch)
return path.Join(c.Repo.Root, "dists", distro, "main/binary-"+arch)
}
func (c Conf) PoolPath(distro, arch string) string {
return filepath.Join(c.Repo.Root, "pool/main", distro, arch)
return path.Join(c.Repo.Root, "pool/main", distro, arch)
}
func (c Conf) PoolPackagePath(distro, arch, name string) string {
return path.Join(c.Repo.Root, c.RelativePoolPackagePath(distro, arch, name))
}
func (c Conf) RelativePoolPackagePath(distro, arch, name string) string {
name = packageName(name)
return filepath.Join(c.Repo.Root, "pool/main", distro, arch, name[0:1], name)
return path.Join(c.Repo.Root, "pool/main", distro, arch, name[0:1], name)
}
func (c RepoConf) DistroNames() []string {

113
src/meow.tf/deb-simple/http.go
View File

@ -4,14 +4,18 @@ import (
"net/http"
"io"
"os"
"path/filepath"
"log"
"encoding/json"
"fmt"
"github.com/blang/semver"
"strings"
"path"
)
func rescanHandler(w http.ResponseWriter, r *http.Request) {
mutex.Lock()
defer mutex.Unlock()
distroName := r.URL.Query().Get("distro")
if distroName == "" {
@ -27,7 +31,12 @@ func rescanHandler(w http.ResponseWriter, r *http.Request) {
scanInitialPackages(conf, distro)
saveCache(distro)
distros[distroName] = distro
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}
func uploadHandler(w http.ResponseWriter, r *http.Request) {
@ -36,10 +45,10 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
return
}
archType := r.URL.Query().Get("arch")
queryArchType := r.URL.Query().Get("arch")
if archType == "" {
archType = "all"
if queryArchType == "" {
queryArchType = "all"
}
distroName := r.URL.Query().Get("distro")
@ -59,7 +68,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
forceStr := r.URL.Query().Get("force")
if forceStr != "" && forceStr == "true" {
if forceStr == "true" {
force = true
}
@ -80,20 +89,14 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
return
}
packages, exists := distro.Architectures[archType]
if !exists {
httpErrorf(w, "invalid arch: %s", archType)
mutex.RUnlock()
return
}
mutex.RUnlock()
// Lock to prevent concurrent modification
mutex.Lock()
defer mutex.Unlock()
modifiedArches := make(map[string]bool)
for {
part, err := reader.NextPart()
@ -101,10 +104,51 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
break
}
if part.FileName() == "" {
if part.FileName() == "" || !strings.HasSuffix(part.FileName(), ".deb"){
continue
}
newPath := conf.PoolPackagePath(distroName, archType, part.FileName())
tempFile := path.Join(os.TempDir(), "temp_" + part.FileName())
dst, err := os.Create(tempFile)
if err != nil {
httpErrorf(w, "error creating deb file: %s", err)
return
}
if _, err := io.Copy(dst, part); err != nil {
dst.Close()
httpErrorf(w, "error writing deb file: %s", err)
return
}
dst.Close()
// Get package name, if it already exists remove the old file.
f, err := newPackageFile(tempFile, part.FileName())
if err != nil {
httpErrorf(w, "error loading package info: %s", err)
}
archType := f.Info.Architecture
if archType == "" {
archType = queryArchType
}
// Get current packages
packages, exists := distro.Architectures[archType]
if !exists {
httpErrorf(w, "invalid arch: %s", archType)
return
}
// New path based off package data
newPath := conf.PoolPackagePath(distroName, archType, f.Info.Package)
if _, err := os.Stat(newPath); err != nil {
if os.IsNotExist(err) {
@ -115,27 +159,13 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
}
}
dst, err := os.Create(filepath.Join(newPath, part.FileName()))
f.Path = path.Join(conf.RelativePoolPackagePath(distroName, archType, f.Info.Package), part.FileName())
if err != nil {
httpErrorf(w, "error creating deb file: %s", err)
if err := os.Rename(tempFile, path.Join(newPath, part.FileName())); err != nil {
httpErrorf(w, "error moving temporary file: %s", err)
return
}
defer dst.Close()
if _, err := io.Copy(dst, part); err != nil {
httpErrorf(w, "error writing deb file: %s", err)
return
}
// Get package name, if it already exists remove the old file.
f, err := newPackageFile(newPath, part.FileName())
if err != nil {
httpErrorf(w, "error loading package info: %s", err)
}
if p, exists := packages[f.Info.Package]; exists {
v1, err1 := semver.Parse(p.Info.Version)
v2, err2 := semver.Parse(f.Info.Version)
@ -149,7 +179,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
// Archive old file
log.Println("Replacing", p.Name, "with", f.Name)
if err := os.Remove(p.Path); err != nil {
if err := os.Remove(path.Join(conf.Repo.Root, p.Path)); err != nil {
httpErrorf(w, "Unable to remove old package: %s", err)
return
}
@ -162,9 +192,11 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
// Recreate the package index and release file.
if err := createPackagesCached(conf, distroName, archType, packages); err != nil {
httpErrorf(w, "error creating package: %s", err)
return
for archType, _ := range modifiedArches {
if err := createPackagesCached(conf, distroName, archType, distro.Architectures[archType]); err != nil {
httpErrorf(w, "error creating package: %s", err)
return
}
}
err = createRelease(conf, distroName)
@ -174,6 +206,13 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
return
}
err = saveCache(distro)
if err != nil {
httpErrorf(w, "error updating cache: %s", err)
return
}
w.WriteHeader(http.StatusCreated)
}
@ -216,7 +255,7 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) {
return
}
debPath := filepath.Join(conf.ArchPath(req.DistroName, req.Arch), req.Filename)
debPath := path.Join(conf.ArchPath(req.DistroName, req.Arch), req.Filename)
if err := os.Remove(debPath); err != nil {
httpErrorf(w, "failed to delete: %s", err)

28
src/meow.tf/deb-simple/packages.go
View File

@ -3,7 +3,6 @@ package simple
import (
"regexp"
"fmt"
"path/filepath"
"strings"
"os"
"bytes"
@ -18,6 +17,7 @@ import (
"github.com/blang/semver"
"encoding/json"
"meow.tf/deb-simple/deb/archive"
"path"
)
var (
@ -68,7 +68,7 @@ var (
)
func loadCache(dist string) error {
f, err := os.Open(filepath.Join(conf.DistPath(dist), "dist.json"))
f, err := os.Open(path.Join(conf.DistPath(dist), "dist.json"))
if err != nil {
return err
@ -88,7 +88,7 @@ func loadCache(dist string) error {
}
func saveCache(dist *Distro) error {
f, err := os.Create(filepath.Join(conf.DistPath(dist.Name), "dist.json"))
f, err := os.Create(path.Join(conf.DistPath(dist.Name), "dist.json"))
if err != nil {
return err
@ -113,8 +113,8 @@ func buildPackageList(config Conf, distro, arch string) (map[string]*PackageFile
return m, nil
}
func scanRecursive(path string, m map[string]*PackageFile) error {
dirList, err := ioutil.ReadDir(path)
func scanRecursive(base string, m map[string]*PackageFile) error {
dirList, err := ioutil.ReadDir(base)
if err != nil {
return err
@ -122,7 +122,7 @@ func scanRecursive(path string, m map[string]*PackageFile) error {
for _, file := range dirList {
if file.IsDir() {
if err := scanRecursive(filepath.Join(path, file.Name()), m); err != nil {
if err := scanRecursive(path.Join(base, file.Name()), m); err != nil {
return err
}
continue
@ -132,7 +132,7 @@ func scanRecursive(path string, m map[string]*PackageFile) error {
continue
}
p, err := newPackageFile(path, file.Name())
p, err := newPackageFile(base, file.Name())
if err != nil {
return err
@ -154,13 +154,13 @@ func scanRecursive(path string, m map[string]*PackageFile) error {
return nil
}
func newPackageFile(path, name string) (*PackageFile, error) {
func newPackageFile(base, name string) (*PackageFile, error) {
p := &PackageFile{
Name: name,
Path: filepath.Join(path, name),
Path: path.Join(stripPrefix(base, conf.Repo.Root), name),
}
debPath := filepath.Join(path, name)
debPath := path.Join(base, name)
var err error
@ -202,13 +202,13 @@ func newPackageFile(path, name string) (*PackageFile, error) {
}
func createPackagesCached(config Conf, distro, arch string, packages map[string]*PackageFile) error {
stdFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages"))
stdFile, err := os.Create(path.Join(config.ArchPath(distro, arch), "Packages"))
if err != nil {
return fmt.Errorf("failed to create packages: %s", err)
}
defer stdFile.Close()
gzipFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages.gz"))
gzipFile, err := os.Create(path.Join(config.ArchPath(distro, arch), "Packages.gz"))
if err != nil {
return fmt.Errorf("failed to create packages.gz: %s", err)
}
@ -227,9 +227,9 @@ func createPackagesCached(config Conf, distro, arch string, packages map[string]
packBuf.WriteString(p.ControlData)
dir := filepath.Join("pool/main", distro, arch, p.Info.Package[0:1], p.Info.Package, p.Name)
dir := path.Join("pool/main", distro, arch, p.Info.Package[0:1], p.Info.Package, p.Name)
fmt.Fprintf(&packBuf, "Filename: %s\n", strings.Replace(dir, "\\", "/", -1))
fmt.Fprintf(&packBuf, "Filename: %s\n", dir)
fmt.Fprintf(&packBuf, "Size: %d\n", p.Size)
fmt.Fprintf(&packBuf, "MD5sum: %s\n", p.MD5Hash)

13
src/meow.tf/deb-simple/server.go
View File

@ -15,7 +15,7 @@ import (
"github.com/go-ini/ini"
)
var VERSION string = "1.2.0"
var VERSION string = "1.3.0"
func packageName(name string) string {
if index := strings.Index(name, "_"); index != -1 {
@ -109,9 +109,11 @@ func scanInitialPackages(config Conf, dist *Distro) {
log.Println("Generating packages file for", dist.Name, arch)
createPackagesCached(config, dist.Name, arch, files)
createRelease(config, dist.Name)
}
log.Println("Generating Release for", dist.Name)
createRelease(config, dist.Name)
saveCache(dist)
}
@ -182,3 +184,10 @@ func createDirs(config Conf) error {
}
return nil
}
func stripPrefix(str, prefix string) string {
if strings.Index(str, prefix) == 0 {
return strings.TrimLeft(str[len(prefix):], "/")
}
return str
}