From a93947ff81ce4babcf60d19e6bbab65246bf23b1 Mon Sep 17 00:00:00 2001 From: Tyler Date: Sun, 11 Jun 2017 03:18:59 -0400 Subject: [PATCH 1/4] Move data into src, cached package structure --- .gitignore | 2 +- .gitlab-ci.yml | 16 - main.go | 176 +----- main_test.go | 535 ------------------ packages.go | 31 - apt.go => src/meow.tf/deb-simple/apt.go | 118 +--- .../meow.tf/deb-simple/glide.lock | 8 +- .../meow.tf/deb-simple/glide.yaml | 2 + http.go => src/meow.tf/deb-simple/http.go | 95 +++- src/meow.tf/deb-simple/packages.go | 213 +++++++ src/meow.tf/deb-simple/server.go | 200 +++++++ 11 files changed, 513 insertions(+), 883 deletions(-) delete mode 100644 .gitlab-ci.yml delete mode 100644 main_test.go delete mode 100644 packages.go rename apt.go => src/meow.tf/deb-simple/apt.go (58%) rename glide.lock => src/meow.tf/deb-simple/glide.lock (51%) rename glide.yaml => src/meow.tf/deb-simple/glide.yaml (68%) rename http.go => src/meow.tf/deb-simple/http.go (63%) create mode 100644 src/meow.tf/deb-simple/packages.go create mode 100644 src/meow.tf/deb-simple/server.go diff --git a/.gitignore b/.gitignore index 59f9a7a..504ea6d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ /deb-simple /repo -/src +vendor conf.json *.pprof diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 3b1fe9a..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,16 +0,0 @@ -image: knightswarm/gobuild:1.6 - -before_script: - - go get github.com/tools/godep - - export GOPATH=$(pwd) - -stages: - - build - -compile: - stage: build - script: - - export DEB_SIMPLE_VERSION=`grep "var VERSION" main.go | awk -F\" '{print $2}'` - - godep restore - - chmod +x ./ci/package.sh - - ./ci/package.sh \ No newline at end of file diff --git a/main.go b/main.go index 439ba71..cc447a9 100644 --- a/main.go +++ b/main.go @@ -1,177 +1,7 @@ package main -import ( - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "path/filepath" - "sync" - "strings" - "golang.org/x/crypto/openpgp" - "runtime" - "errors" -) - -var VERSION string = "1.1.0" - -type Conf struct { - ListenPort string `json:"listenPort"` - RootRepoPath string `json:"rootRepoPath"` - SupportArch []string `json:"supportedArch"` - DistroNames []string `json:"distroNames"` - PGPSecretKey string `json:"pgpSecretKey"` - PGPPassphrase string `json:"pgpPassphrase"` - EnableSSL bool `json:"enableSSL"` - SSLCert string `json:"SSLcert"` - SSLKey string `json:"SSLkey"` - Key string `json:"key"` -} - -func (c Conf) DistPath(distro string) string { - return filepath.Join(c.RootRepoPath, "dists", distro) -} - -func (c Conf) ArchPath(distro, arch string) string { - return filepath.Join(c.RootRepoPath, "dists", distro, "main/binary-"+arch) -} - -func (c Conf) PoolPath(distro, arch string) string { - return filepath.Join(c.RootRepoPath, "pool/main", distro, arch) -} - -func (c Conf) PoolPackagePath(distro, arch, name string) string { - name = packageName(name) - return filepath.Join(c.RootRepoPath, "pool/main", distro, arch, name[0:1], name) -} - -func packageName(name string) string { - if index := strings.Index(name, "_"); index != -1 { - name = name[:index] - } - return name -} - -type DeleteObj struct { - Filename string - DistroName string - Arch string -} - -var ( - mutex sync.Mutex - configFile = flag.String("c", "conf.json", "config file location") - flagShowVersion = flag.Bool("version", false, "Show dnsconfig version") - parsedConfig = Conf{} - pgpEntity *openpgp.Entity -) +import "meow.tf/deb-simple" func main() { - flag.Parse() - - if *flagShowVersion { - fmt.Printf("deb-simple %s (%s)\n", VERSION, runtime.Version()) - os.Exit(0) - } - - file, err := ioutil.ReadFile(*configFile) - if err != nil { - log.Fatalln("unable to read config file, exiting...") - } - if err := json.Unmarshal(file, &parsedConfig); err != nil { - log.Fatalln("unable to marshal config file, exiting...") - } - - if err := createDirs(parsedConfig); err != nil { - log.Println(err) - log.Fatalln("error creating directory structure, exiting") - } - - if err := setupPgp(parsedConfig); err != nil { - log.Println(err) - log.Fatalln("error loading pgp key, exiting") - } - - http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(parsedConfig.RootRepoPath)))) - http.Handle("/upload", uploadHandler(parsedConfig)) - http.Handle("/delete", deleteHandler(parsedConfig)) - - if parsedConfig.EnableSSL { - log.Println("running with SSL enabled") - log.Fatalln(http.ListenAndServeTLS(":"+parsedConfig.ListenPort, parsedConfig.SSLCert, parsedConfig.SSLKey, nil)) - } else { - log.Println("running without SSL enabled") - log.Fatalln(http.ListenAndServe(":"+parsedConfig.ListenPort, nil)) - } -} - -func setupPgp(config Conf) error { - if config.PGPSecretKey == "" { - return nil - } - - secretKey, err := os.Open(config.PGPSecretKey) - if err != nil { - return fmt.Errorf("failed to open private key ring file: %s", err) - } - defer secretKey.Close() - - entitylist, err := openpgp.ReadKeyRing(secretKey) - - if err != nil { - return fmt.Errorf("failed to read key ring: %s", err) - } - - if len(entitylist) < 1 { - return errors.New("no keys in key ring") - } - - pgpEntity = entitylist[0] - - passphrase := []byte(config.PGPPassphrase) - - if err := pgpEntity.PrivateKey.Decrypt(passphrase); err != nil { - return err - } - - for _, subkey := range pgpEntity.Subkeys { - err := subkey.PrivateKey.Decrypt(passphrase) - - if err != nil { - return err - } - } - - return nil -} - -func createDirs(config Conf) error { - for _, distro := range config.DistroNames { - for _, arch := range config.SupportArch { - if _, err := os.Stat(config.ArchPath(distro, arch)); err != nil { - if os.IsNotExist(err) { - log.Printf("Directory for %s (%s) does not exist, creating", distro, arch) - if err := os.MkdirAll(config.ArchPath(distro, arch), 0755); err != nil { - return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err) - } - } else { - return fmt.Errorf("error inspecting %s (%s): %s", distro, arch, err) - } - } - if _, err := os.Stat(config.PoolPath(distro, arch)); err != nil { - if os.IsNotExist(err) { - log.Printf("Directory for %s (%s) does not exist, creating", distro, arch) - if err := os.MkdirAll(config.PoolPath(distro, arch), 0755); err != nil { - return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err) - } - } else { - return fmt.Errorf("error inspecting %s (%s): %s", distro, arch, err) - } - } - } - } - return nil -} + simple.Start() +} \ No newline at end of file diff --git a/main_test.go b/main_test.go deleted file mode 100644 index b383da9..0000000 --- a/main_test.go +++ /dev/null @@ -1,535 +0,0 @@ -package main - -import ( - "bytes" - "compress/gzip" - "crypto/md5" - "encoding/hex" - "io" - "io/ioutil" - "mime/multipart" - "net/http" - "net/http/httptest" - "os" - "testing" -) - -var goodOutput = `Package: vim-tiny -Source: vim -Version: 2:7.4.052-1ubuntu3 -Architecture: amd64 -Maintainer: Ubuntu Developers -Installed-Size: 931 -Depends: vim-common (= 2:7.4.052-1ubuntu3), libacl1 (>= 2.2.51-8), libc6 (>= 2.15), libselinux1 (>= 1.32), libtinfo5 -Suggests: indent -Provides: editor -Section: editors -Priority: important -Homepage: http://www.vim.org/ -Description: Vi IMproved - enhanced vi editor - compact version - Vim is an almost compatible version of the UNIX editor Vi. - . - Many new features have been added: multi level undo, syntax - highlighting, command line history, on-line help, filename - completion, block operations, folding, Unicode support, etc. - . - This package contains a minimal version of vim compiled with no - GUI and a small subset of features in order to keep small the - package size. This package does not depend on the vim-runtime - package, but installing it you will get its additional benefits - (online documentation, plugins, ...). -Original-Maintainer: Debian Vim Maintainers -` - -var goodPkgGzOutput = `Package: vim-tiny -Source: vim -Version: 2:7.4.052-1ubuntu3 -Architecture: amd64 -Maintainer: Ubuntu Developers -Installed-Size: 931 -Depends: vim-common (= 2:7.4.052-1ubuntu3), libacl1 (>= 2.2.51-8), libc6 (>= 2.15), libselinux1 (>= 1.32), libtinfo5 -Suggests: indent -Provides: editor -Section: editors -Priority: important -Homepage: http://www.vim.org/ -Description: Vi IMproved - enhanced vi editor - compact version - Vim is an almost compatible version of the UNIX editor Vi. - . - Many new features have been added: multi level undo, syntax - highlighting, command line history, on-line help, filename - completion, block operations, folding, Unicode support, etc. - . - This package contains a minimal version of vim compiled with no - GUI and a small subset of features in order to keep small the - package size. This package does not depend on the vim-runtime - package, but installing it you will get its additional benefits - (online documentation, plugins, ...). -Original-Maintainer: Debian Vim Maintainers -Filename: dists/stable/main/binary-cats/test.deb -Size: 391240 -MD5sum: 0ec79417129746ff789fcff0976730c5 -SHA1: b2ac976af80f0f50a8336402d5a29c67a2880b9b -SHA256: 9938ec82a8c882ebc2d59b64b0bf2ac01e9cbc5a235be4aa268d4f8484e75eab - - -` - -var goodPkgGzOutputNonDefault = `Package: vim-tiny -Source: vim -Version: 2:7.4.052-1ubuntu3 -Architecture: amd64 -Maintainer: Ubuntu Developers -Installed-Size: 931 -Depends: vim-common (= 2:7.4.052-1ubuntu3), libacl1 (>= 2.2.51-8), libc6 (>= 2.15), libselinux1 (>= 1.32), libtinfo5 -Suggests: indent -Provides: editor -Section: editors -Priority: important -Homepage: http://www.vim.org/ -Description: Vi IMproved - enhanced vi editor - compact version - Vim is an almost compatible version of the UNIX editor Vi. - . - Many new features have been added: multi level undo, syntax - highlighting, command line history, on-line help, filename - completion, block operations, folding, Unicode support, etc. - . - This package contains a minimal version of vim compiled with no - GUI and a small subset of features in order to keep small the - package size. This package does not depend on the vim-runtime - package, but installing it you will get its additional benefits - (online documentation, plugins, ...). -Original-Maintainer: Debian Vim Maintainers -Filename: dists/blah/main/binary-cats/test.deb -Size: 391240 -MD5sum: 0ec79417129746ff789fcff0976730c5 -SHA1: b2ac976af80f0f50a8336402d5a29c67a2880b9b -SHA256: 9938ec82a8c882ebc2d59b64b0bf2ac01e9cbc5a235be4aa268d4f8484e75eab - - -` - -func TestCreateDirs(t *testing.T) { - pwd, err := os.Getwd() - if err != nil { - t.Errorf("Unable to get current working directory: %s", err) - } - config := Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"stable"}, EnableSSL: false} - // sanity check... - if config.RootRepoPath != pwd+"/testing" { - t.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing") - } - t.Log("creating temp dirs in ", config.RootRepoPath) - if err := createDirs(config); err != nil { - t.Errorf("createDirs() failed ") - } - for _, archDir := range config.SupportArch { - if _, err := os.Stat(config.RootRepoPath + "/dists/stable/main/binary-" + archDir); err != nil { - if os.IsNotExist(err) { - t.Errorf("Directory for %s does not exist", archDir) - } - } - } - - // cleanup - if err := os.RemoveAll(config.RootRepoPath); err != nil { - t.Errorf("error cleaning up after createDirs(): %s", err) - } - - // create temp file - tempFile, err := os.Create(pwd + "/tempFile") - if err != nil { - t.Fatalf("create %s: %s", pwd+"/tempFile", err) - } - defer tempFile.Close() - config.RootRepoPath = pwd + "/tempFile" - // Can't make directory named after file. - if err := createDirs(config); err == nil { - t.Errorf("createDirs() should have failed but did not") - } - // cleanup - if err := os.RemoveAll(pwd + "/tempFile"); err != nil { - t.Errorf("error cleaning up after createDirs(): %s", err) - } - -} - -func TestInspectPackage(t *testing.T) { - parsedControl, err := inspectPackage("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb") - if err != nil { - t.Error("inspectPackage() error: %s", err) - } - if parsedControl != goodOutput { - t.Errorf("control file does not match") - } - - _, err = inspectPackage("thisfileshouldnotexist") - if err == nil { - t.Error("inspectPackage() should have failed, it did not") - } -} - -func TestInspectPackageControl(t *testing.T) { - sampleDeb, err := ioutil.ReadFile("samples/control.tar.gz") - if err != nil { - t.Errorf("error opening sample deb file: %s", err) - } - var controlBuf bytes.Buffer - cfReader := bytes.NewReader(sampleDeb) - io.Copy(&controlBuf, cfReader) - parsedControl, err := inspectPackageControl(controlBuf) - if err != nil { - t.Error("error inspecting control file: %s", err) - } - if parsedControl != goodOutput { - t.Errorf("control file does not match") - } - - var failControlBuf bytes.Buffer - _, err = inspectPackageControl(failControlBuf) - if err == nil { - t.Error("inspectPackageControl() should have failed, it did not") - } - -} - -func TestCreatePackagesGz(t *testing.T) { - pwd, err := os.Getwd() - if err != nil { - t.Errorf("Unable to get current working directory: %s", err) - } - config := Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"stable"}, EnableSSL: false} - // sanity check... - if config.RootRepoPath != pwd+"/testing" { - t.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing") - } - // copy sample deb to repo location (assuming it exists) - origDeb, err := os.Open("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb") - if err != nil { - t.Errorf("error opening up sample deb: %s", err) - } - defer origDeb.Close() - for _, archDir := range config.SupportArch { - // do not use the built-in createDirs() in case it is broken - if err := os.MkdirAll(config.RootRepoPath+"/dists/stable/main/binary-"+archDir, 0755); err != nil { - t.Errorf("error creating directory for %s: %s\n", archDir, err) - } - copyDeb, err := os.Create(config.RootRepoPath + "/dists/stable/main/binary-" + archDir + "/test.deb") - if err != nil { - t.Errorf("error creating copy of deb: %s", err) - } - _, err = io.Copy(copyDeb, origDeb) - if err != nil { - t.Errorf("error writing copy of deb: %s", err) - } - if err := copyDeb.Close(); err != nil { - t.Errorf("error saving copy of deb: %s", err) - } - } - if err := createPackagesGz(config, "stable", "cats"); err != nil { - t.Errorf("error creating packages gzip for cats") - } - pkgGzip, err := ioutil.ReadFile(config.RootRepoPath + "/dists/stable/main/binary-cats/Packages.gz") - if err != nil { - t.Errorf("error reading Packages.gz: %s", err) - } - pkgReader, err := gzip.NewReader(bytes.NewReader(pkgGzip)) - if err != nil { - t.Errorf("error reading existing Packages.gz: %s", err) - } - buf := bytes.NewBuffer(nil) - io.Copy(buf, pkgReader) - if goodPkgGzOutput != string(buf.Bytes()) { - t.Errorf("Packages.gz does not match, returned value is: %s", string(buf.Bytes())) - } - - // cleanup - if err := os.RemoveAll(config.RootRepoPath); err != nil { - t.Errorf("error cleaning up after createPackagesGz(): %s", err) - } - - // create temp file - tempFile, err := os.Create(pwd + "/tempFile") - if err != nil { - t.Fatalf("create %s: %s", pwd+"/tempFile", err) - } - defer tempFile.Close() - config.RootRepoPath = pwd + "/tempFile" - // Can't make directory named after file - if err := createPackagesGz(config, "stable", "cats"); err == nil { - t.Errorf("createPackagesGz() should have failed, it did not") - } - // cleanup - if err := os.RemoveAll(pwd + "/tempFile"); err != nil { - t.Errorf("error cleaning up after createDirs(): %s", err) - } - -} - -func TestCreatePackagesGzNonDefault(t *testing.T) { - pwd, err := os.Getwd() - if err != nil { - t.Errorf("Unable to get current working directory: %s", err) - } - config := Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"blah"}, EnableSSL: false} - // sanity check... - if config.RootRepoPath != pwd+"/testing" { - t.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing") - } - // copy sample deb to repo location (assuming it exists) - origDeb, err := os.Open("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb") - if err != nil { - t.Errorf("error opening up sample deb: %s", err) - } - defer origDeb.Close() - for _, archDir := range config.SupportArch { - // do not use the built-in createDirs() in case it is broken - if err := os.MkdirAll(config.RootRepoPath+"/dists/blah/main/binary-"+archDir, 0755); err != nil { - t.Errorf("error creating directory for %s: %s\n", archDir, err) - } - copyDeb, err := os.Create(config.RootRepoPath + "/dists/blah/main/binary-" + archDir + "/test.deb") - if err != nil { - t.Errorf("error creating copy of deb: %s", err) - } - _, err = io.Copy(copyDeb, origDeb) - if err != nil { - t.Errorf("error writing copy of deb: %s", err) - } - if err := copyDeb.Close(); err != nil { - t.Errorf("error saving copy of deb: %s", err) - } - } - if err := createPackagesGz(config, "blah", "cats"); err != nil { - t.Errorf("error creating packages gzip for cats") - } - pkgGzip, err := ioutil.ReadFile(config.RootRepoPath + "/dists/blah/main/binary-cats/Packages.gz") - if err != nil { - t.Errorf("error reading Packages.gz: %s", err) - } - pkgReader, err := gzip.NewReader(bytes.NewReader(pkgGzip)) - if err != nil { - t.Errorf("error reading existing Packages.gz: %s", err) - } - buf := bytes.NewBuffer(nil) - io.Copy(buf, pkgReader) - if goodPkgGzOutputNonDefault != string(buf.Bytes()) { - t.Errorf("Packages.gz does not match, returned value is: %s", string(buf.Bytes())) - } - - // cleanup - if err := os.RemoveAll(config.RootRepoPath); err != nil { - t.Errorf("error cleaning up after createPackagesGz(): %s", err) - } - -} - -func TestUploadHandler(t *testing.T) { - pwd, err := os.Getwd() - if err != nil { - t.Errorf("Unable to get current working directory: %s", err) - } - config := Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"stable"}, EnableSSL: false} - // sanity check... - if config.RootRepoPath != pwd+"/testing" { - t.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing") - } - uploadHandle := uploadHandler(config) - // GET - req, _ := http.NewRequest("GET", "", nil) - w := httptest.NewRecorder() - uploadHandle.ServeHTTP(w, req) - if w.Code != http.StatusMethodNotAllowed { - t.Errorf("uploadHandler GET returned %v, should be %v", w.Code, http.StatusMethodNotAllowed) - } - - // POST - // create "all" arch as it's the default - if err := os.MkdirAll(config.RootRepoPath+"/dists/stable/main/binary-all", 0755); err != nil { - t.Error("error creating directory for POST testing: %s", err) - } - sampleDeb, err := os.Open("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb") - if err != nil { - t.Errorf("error opening sample deb file: %s", err) - } - defer sampleDeb.Close() - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - part, err := writer.CreateFormFile("file", "vim-tiny_7.4.052-1ubuntu3_amd64.deb") - if err != nil { - t.Errorf("error FormFile: %s", err) - } - _, err = io.Copy(part, sampleDeb) - if err != nil { - t.Errorf("error copying sampleDeb to FormFile: %s", err) - } - if err := writer.Close(); err != nil { - t.Errorf("error closing form writer: %s", err) - } - req, _ = http.NewRequest("POST", "", body) - req.Header.Add("Content-Type", writer.FormDataContentType()) - w = httptest.NewRecorder() - uploadHandle.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Errorf("uploadHandler POST returned %v, should be %v", w.Code, http.StatusOK) - } - // verify uploaded file matches sample - uploadFile, _ := ioutil.ReadFile(config.RootRepoPath + "/dists/stable/main/binary-all/vim-tiny_7.4.052-1ubuntu3_amd64.deb") - uploadmd5hash := md5.New() - uploadmd5hash.Write(uploadFile) - uploadFilemd5 := hex.EncodeToString(uploadmd5hash.Sum(nil)) - - sampleFile, _ := ioutil.ReadFile("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb") - samplemd5hash := md5.New() - samplemd5hash.Write(sampleFile) - sampleFilemd5 := hex.EncodeToString(samplemd5hash.Sum(nil)) - if uploadFilemd5 != sampleFilemd5 { - t.Errorf("uploaded file MD5 is %s, should be %s", uploadFilemd5, sampleFilemd5) - } - - // cleanup - if err := os.RemoveAll(config.RootRepoPath); err != nil { - t.Errorf("error cleaning up after uploadHandler(): %s", err) - } - - // create temp file - tempFile, err := os.Create(pwd + "/tempFile") - if err != nil { - t.Fatalf("create %s: %s", pwd+"/tempFile", err) - } - defer tempFile.Close() - config.RootRepoPath = pwd + "/tempFile" - // Can't make directory named after file - uploadHandle = uploadHandler(config) - failBody := &bytes.Buffer{} - failWriter := multipart.NewWriter(failBody) - failPart, err := failWriter.CreateFormFile("file", "vim-tiny_7.4.052-1ubuntu3_amd64.deb") - if err != nil { - t.Errorf("error FormFile: %s", err) - } - _, err = io.Copy(failPart, sampleDeb) - if err != nil { - t.Errorf("error copying sampleDeb to FormFile: %s", err) - } - if err := failWriter.Close(); err != nil { - t.Errorf("error closing form writer: %s", err) - } - req, _ = http.NewRequest("POST", "", failBody) - req.Header.Add("Content-Type", failWriter.FormDataContentType()) - w = httptest.NewRecorder() - uploadHandle.ServeHTTP(w, req) - if w.Code != http.StatusInternalServerError { - t.Errorf("uploadHandler POST returned %v, should be %v", w.Code, http.StatusInternalServerError) - } - // cleanup - if err := os.RemoveAll(pwd + "/tempFile"); err != nil { - t.Errorf("error cleaning up after createDirs(): %s", err) - } -} - -func TestDeleteHandler(t *testing.T) { - pwd, err := os.Getwd() - if err != nil { - t.Errorf("Unable to get current working directory: %s", err) - } - config := Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"stable"}, EnableSSL: false} - // sanity check... - if config.RootRepoPath != pwd+"/testing" { - t.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing") - } - deleteHandle := deleteHandler(config) - // GET - req, _ := http.NewRequest("GET", "", nil) - w := httptest.NewRecorder() - deleteHandle.ServeHTTP(w, req) - if w.Code != http.StatusMethodNotAllowed { - t.Errorf("deleteHandler GET returned %v, should be %v", w.Code, http.StatusMethodNotAllowed) - } - - // DELETE - // create "all" arch as it's the default - if err := os.MkdirAll(config.RootRepoPath+"/dists/stable/main/binary-all", 0755); err != nil { - t.Error("error creating directory for POST testing: %s", err) - } - tempDeb, err := os.Create(config.RootRepoPath + "/dists/stable/main/binary-all/myapp.deb") - if err != nil { - t.Fatalf("create %s: %s", config.RootRepoPath+"/dists/stable/main/binary-all/myapp.deb", err) - } - defer tempDeb.Close() - req, _ = http.NewRequest("DELETE", "", bytes.NewBufferString("{\"filename\":\"myapp.deb\",\"arch\":\"all\", \"distroName\":\"stable\"}")) - w = httptest.NewRecorder() - deleteHandle.ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Errorf("deleteHandler DELETE returned %v, should be %v", w.Code, http.StatusOK) - } - - // cleanup - if err := os.RemoveAll(config.RootRepoPath); err != nil { - t.Errorf("error cleaning up after uploadHandler(): %s", err) - } - - // create temp file - tempFile, err := os.Create(pwd + "/tempFile") - if err != nil { - t.Fatalf("create %s: %s", pwd+"/tempFile", err) - } - defer tempFile.Close() - config.RootRepoPath = pwd + "/tempFile" - // Can't make directory named after file - deleteHandle = deleteHandler(config) - req, _ = http.NewRequest("DELETE", "", bytes.NewBufferString("{\"filename\":\"myapp.deb\",\"arch\":\"amd64\", \"distroName\":\"stable\"}")) - w = httptest.NewRecorder() - deleteHandle.ServeHTTP(w, req) - if w.Code != http.StatusInternalServerError { - t.Errorf("deleteHandler DELETE returned %v, should be %v", w.Code, http.StatusInternalServerError) - } - // cleanup - if err := os.RemoveAll(pwd + "/tempFile"); err != nil { - t.Errorf("error cleaning up after createDirs(): %s", err) - } -} - -func BenchmarkUploadHandler(b *testing.B) { - pwd, err := os.Getwd() - if err != nil { - b.Errorf("Unable to get current working directory: %s", err) - } - config := &Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"stable"}, EnableSSL: false} - // sanity check... - if config.RootRepoPath != pwd+"/testing" { - b.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing") - } - uploadHandle := uploadHandler(*config) - if err := os.MkdirAll(config.RootRepoPath+"/dists/stable/main/binary-all", 0755); err != nil { - b.Errorf("error creating directory for POST testing: %s", err) - } - sampleDeb, err := os.Open("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb") - if err != nil { - b.Errorf("error opening sample deb file: %s", err) - } - defer sampleDeb.Close() - b.ResetTimer() - for i := 0; i < b.N; i++ { - // temporary (i hope) hack to solve "http: MultipartReader called twice" error - b.StopTimer() - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - part, err := writer.CreateFormFile("file", "vim-tiny_7.4.052-1ubuntu3_amd64.deb") - if err != nil { - b.Errorf("error FormFile: %s", err) - } - if _, err := io.Copy(part, sampleDeb); err != nil { - b.Errorf("error copying sampleDeb to FormFile: %s", err) - } - if err := writer.Close(); err != nil { - b.Errorf("error closing form writer: %s", err) - } - req, _ := http.NewRequest("POST", "/upload?distro=stable", body) - req.Header.Add("Content-Type", writer.FormDataContentType()) - w := httptest.NewRecorder() - b.StartTimer() - uploadHandle.ServeHTTP(w, req) - } - b.StopTimer() - // cleanup - _ = os.RemoveAll(config.RootRepoPath) -} diff --git a/packages.go b/packages.go deleted file mode 100644 index 6a51f02..0000000 --- a/packages.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "regexp" - "log" - "reflect" -) - -var ( - variableRegexp = regexp.MustCompile("(.*?):\\s*(.*)") -) - -type Package struct { - Package string - Version string -} - -func parsePackageData(ctlData string) (*Package, error) { - res := &Package{} - - for _, match := range variableRegexp.FindAllStringSubmatch(ctlData, -1) { - switch match[1] { - case "Package": - res.Package = match[2] - case "Version": - res.Version = match[2] - } - } - - return res, nil -} \ No newline at end of file diff --git a/apt.go b/src/meow.tf/deb-simple/apt.go similarity index 58% rename from apt.go rename to src/meow.tf/deb-simple/apt.go index 757c21a..ca78344 100644 --- a/apt.go +++ b/src/meow.tf/deb-simple/apt.go @@ -1,4 +1,4 @@ -package main +package simple import ( "os" @@ -17,7 +17,6 @@ import ( "crypto/sha256" "encoding/hex" "golang.org/x/crypto/openpgp" - "bufio" "time" ) @@ -89,125 +88,14 @@ func inspectPackageControl(filename bytes.Buffer) (string, error) { return "", nil } -func createPackagesGz(config Conf, distro, arch string) error { - stdFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages")) - if err != nil { - return fmt.Errorf("failed to create packages: %s", err) - } - defer stdFile.Close() - - gzipFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages.gz")) - if err != nil { - return fmt.Errorf("failed to create packages.gz: %s", err) - } - defer gzipFile.Close() - - gzWriter := gzip.NewWriter(gzipFile) - defer gzWriter.Close() - - stdOut := bufio.NewWriter(stdFile) - - // loop through each directory - // run inspectPackage - - poolPath := config.PoolPath(distro, arch) - - dirList, err := ioutil.ReadDir(poolPath) - if err != nil { - return fmt.Errorf("scanning: %s: %s", poolPath, err) - } - for _, firstChar := range dirList { - // Recursing isn't fun... - dirList, err := ioutil.ReadDir(filepath.Join(poolPath, firstChar.Name())) - if err != nil { - return fmt.Errorf("scanning: %s: %s", poolPath, err) - } - - for _, p := range dirList { - dirList, err := ioutil.ReadDir(filepath.Join(poolPath, firstChar.Name(), p.Name())) - - if err != nil { - return fmt.Errorf("scanning: %s: %s", poolPath, err) - } - - for _, debFile := range dirList { - if strings.HasSuffix(debFile.Name(), "deb") { - var packBuf bytes.Buffer - debName := packageName(debFile.Name()) - debPath := filepath.Join(config.PoolPackagePath(distro, arch, debFile.Name()), debFile.Name()) - tempCtlData, err := inspectPackage(debPath) - if err != nil { - return err - } - packBuf.WriteString(tempCtlData) - - var size int64 = debFile.Size() - - if size == 0 { - if stat, err := os.Stat(debPath); err == nil { - size = stat.Size() - } - } - - dir := filepath.Join("pool/main", distro, arch, debName[0:1], debName, debFile.Name()) - log.Println("path:", dir) - fmt.Fprintf(&packBuf, "Filename: %s\n", strings.Replace(dir, "\\", "/", -1)) - fmt.Fprintf(&packBuf, "Size: %d\n", size) - - f, err := os.Open(debPath) - - if err != nil { - log.Println("error opening deb file: ", err) - } - - var ( - md5hash = md5.New() - sha1hash = sha1.New() - sha256hash = sha256.New() - ) - - f.Seek(0, 0) - if _, err := io.Copy(md5hash, f); err != nil { - log.Println("error with the md5 hash: ", err) - } - fmt.Fprintf(&packBuf, "MD5sum: %s\n", - hex.EncodeToString(md5hash.Sum(nil))) - f.Seek(0, 0) - if _, err = io.Copy(sha1hash, f); err != nil { - log.Println("error with the sha1 hash: ", err) - } - fmt.Fprintf(&packBuf, "SHA1: %s\n", - hex.EncodeToString(sha1hash.Sum(nil))) - f.Seek(0, 0) - if _, err = io.Copy(sha256hash, f); err != nil { - log.Println("error with the sha256 hash: ", err) - } - fmt.Fprintf(&packBuf, "SHA256: %s\n", - hex.EncodeToString(sha256hash.Sum(nil))) - packBuf.WriteString("\n\n") - - stdOut.Write(packBuf.Bytes()) - gzWriter.Write(packBuf.Bytes()) - - f.Close() - f = nil - } - } - } - } - - stdOut.Flush() - - gzWriter.Flush() - - return nil -} func createRelease(config Conf, distro, arch string) error { outfile, err := os.Create(filepath.Join(config.DistPath(distro), "Release")) + if err != nil { return fmt.Errorf("failed to create Release: %s", err) } + defer outfile.Close() var packBuf bytes.Buffer diff --git a/glide.lock b/src/meow.tf/deb-simple/glide.lock similarity index 51% rename from glide.lock rename to src/meow.tf/deb-simple/glide.lock index 15bd8e3..2d46210 100644 --- a/glide.lock +++ b/src/meow.tf/deb-simple/glide.lock @@ -1,10 +1,12 @@ -hash: f49345e0dcd6cb4f3c49af64a95afbea2e608503cd6c1909d300e4a457ef65a6 -updated: 2017-05-02T21:07:03.8312824-04:00 +hash: 79d0f0e99e99ce9318bb0e8c6ab398657a20679dbe3c421c6fc588db7a738131 +updated: 2017-06-11T03:04:15.102617-04:00 imports: - name: github.com/blakesmith/ar version: 8bd4349a67f2533b078dbc524689d15dba0f4659 +- name: github.com/blang/semver + version: b38d23b8782a487059e8fc8773e9a5b228a77cb6 - name: golang.org/x/crypto - version: d1464577745bc7f4e74f65be9cfbd09436a729d6 + version: 7e9105388ebff089b3f99f0ef676ea55a6da3a7e subpackages: - cast5 - openpgp diff --git a/glide.yaml b/src/meow.tf/deb-simple/glide.yaml similarity index 68% rename from glide.yaml rename to src/meow.tf/deb-simple/glide.yaml index 5dd5c7c..eaa6c8e 100644 --- a/glide.yaml +++ b/src/meow.tf/deb-simple/glide.yaml @@ -4,3 +4,5 @@ import: - package: golang.org/x/crypto subpackages: - openpgp +- package: github.com/blang/semver + version: ^3.5.0 diff --git a/http.go b/src/meow.tf/deb-simple/http.go similarity index 63% rename from http.go rename to src/meow.tf/deb-simple/http.go index 45b31c2..9db6e7d 100644 --- a/http.go +++ b/src/meow.tf/deb-simple/http.go @@ -1,4 +1,4 @@ -package main +package simple import ( "net/http" @@ -17,28 +17,62 @@ func uploadHandler(config Conf) http.Handler { return } archType := r.URL.Query().Get("arch") + if archType == "" { archType = "all" } + distroName := r.URL.Query().Get("distro") + if distroName == "" { distroName = "stable" } + key := r.URL.Query().Get("key") + if key == "" || key != config.Key { http.Error(w, "unauthorized", 403) return } + reader, err := r.MultipartReader() + if err != nil { httpErrorf(w, "error creating multipart reader: %s", err) return } + + mutex.RLock() + + distro, exists := distros[distroName] + + if !exists { + httpErrorf(w, "invalid distro: %s", distroName) + mutex.RUnlock() + return + } + + packages, exists := distro.Architectures[archType] + + if !exists { + httpErrorf(w, "invalid arch: %s", archType) + mutex.RUnlock() + return + } + + mutex.RUnlock() + + // Lock to prevent concurrent modification + mutex.Lock() + defer mutex.Unlock() + for { part, err := reader.NextPart() + if err == io.EOF { break } + if part.FileName() == "" { continue } @@ -54,22 +88,40 @@ func uploadHandler(config Conf) http.Handler { } dst, err := os.Create(filepath.Join(newPath, part.FileName())) + if err != nil { httpErrorf(w, "error creating deb file: %s", err) return } + defer dst.Close() if _, err := io.Copy(dst, part); err != nil { httpErrorf(w, "error writing deb file: %s", err) return } + + // Get package name, if it already exists remove the old file. + f, err := newPackageFile(newPath, part.FileName()) + + if err != nil { + httpErrorf(w, "error loading package info: %s", err) + } + + if p, exists := packages[f.Info.Package]; exists { + // Archive old file + log.Println("Replacing", p.Name, "with", f.Name) + os.Remove(p.Path) + } + + packages[f.Info.Package] = f } - mutex.Lock() - defer mutex.Unlock() log.Println("got lock, updating package list...") - if err := createPackagesGz(config, distroName, archType); err != nil { + + // Recreate the package index and release file. + + if err := createPackagesCached(config, distroName, archType, packages); err != nil { httpErrorf(w, "error creating package: %s", err) return } @@ -91,17 +143,42 @@ func deleteHandler(config Conf) http.Handler { http.Error(w, "method not supported", http.StatusMethodNotAllowed) return } - var toDelete DeleteObj - if err := json.NewDecoder(r.Body).Decode(&toDelete); err != nil { + var req DeleteObj + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { httpErrorf(w, "failed to decode json: %s", err) return } - debPath := filepath.Join(config.ArchPath(toDelete.DistroName, toDelete.Arch), toDelete.Filename) + + mutex.RLock() + + distro, exists := distros[req.DistroName] + + if !exists { + httpErrorf(w, "invalid distro: %s", req.DistroName) + mutex.RUnlock() + return + } + + packages, exists := distro.Architectures[req.Arch] + + if !exists { + httpErrorf(w, "invalid arch: %s", req.Arch) + mutex.RUnlock() + return + } + + mutex.RUnlock() + + debPath := filepath.Join(config.ArchPath(req.DistroName, req.Arch), req.Filename) + if err := os.Remove(debPath); err != nil { httpErrorf(w, "failed to delete: %s", err) return } + key := r.URL.Query().Get("key") + if key == "" || key != config.Key { http.Error(w, "unauthorized", http.StatusForbidden) return @@ -110,12 +187,12 @@ func deleteHandler(config Conf) http.Handler { defer mutex.Unlock() log.Println("got lock, updating package list...") - if err := createPackagesGz(config, toDelete.DistroName, toDelete.Arch); err != nil { + if err := createPackagesCached(config, req.DistroName, req.Arch, packages); err != nil { httpErrorf(w, "failed to delete package: %s", err) return } - if err := createRelease(config, toDelete.DistroName, toDelete.Arch); err != nil { + if err := createRelease(config, req.DistroName, req.Arch); err != nil { httpErrorf(w, "failed to delete package: %s", err) return } diff --git a/src/meow.tf/deb-simple/packages.go b/src/meow.tf/deb-simple/packages.go new file mode 100644 index 0000000..81fc1dd --- /dev/null +++ b/src/meow.tf/deb-simple/packages.go @@ -0,0 +1,213 @@ +package simple + +import ( + "regexp" + "fmt" + "path/filepath" + "strings" + "os" + "bytes" + "bufio" + "compress/gzip" + "encoding/hex" + "io" + "io/ioutil" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "log" + "github.com/blang/semver" +) + +var ( + variableRegexp = regexp.MustCompile("(.*?):\\s*(.*)") +) + +type PackageFile struct { + Path string + Name string + Size int64 + MD5Hash string + SHA1Hash string + SHA256Hash string + ControlData string + Info *Package +} + +type Package struct { + Package string + Version string + Architecture string +} + +func parsePackageData(ctlData string) *Package { + res := &Package{} + + for _, match := range variableRegexp.FindAllStringSubmatch(ctlData, -1) { + switch match[1] { + case "Package": + res.Package = match[2] + case "Architecture": + res.Architecture = match[2] + case "Version": + res.Version = match[2] + } + } + + return res +} + +type Distro struct { + Name string + Architectures map[string]map[string]*PackageFile +} + +var ( + distros map[string]*Distro = make(map[string]*Distro) +) + +func buildPackageList(config Conf, distro, arch string) (map[string]*PackageFile, error) { + m := make(map[string]*PackageFile) + + if err := scanRecursive(config.PoolPath(distro, arch), m); err != nil { + return nil, err + } + + return m, nil +} + +func scanRecursive(path string, m map[string]*PackageFile) error { + dirList, err := ioutil.ReadDir(path) + + if err != nil { + return err + } + + for _, file := range dirList { + if file.IsDir() { + if err := scanRecursive(filepath.Join(path, file.Name()), m); err != nil { + return err + } + continue + } + + if !strings.HasSuffix(file.Name(), "deb") { + continue + } + + p, err := newPackageFile(path, file.Name()) + + if err != nil { + return err + } + + if old, exists := m[p.Info.Package]; exists { + v1, err := semver.Parse(old.Info.Version) + v2, err2 := semver.Parse(p.Info.Version) + + if err == nil && err2 == nil && v1.Compare(v2) > 0 { + // Use old version + continue + } + } + + m[p.Info.Package] = p + } + + return nil +} + +func newPackageFile(path, name string) (*PackageFile, error) { + log.Println("New package from", path, name) + p := &PackageFile{ + Name: name, + Path: filepath.Join(path, name), + } + + debPath := filepath.Join(path, name) + + var err error + + p.ControlData, err = inspectPackage(debPath) + + if err != nil { + return nil, err + } + + p.Info = parsePackageData(p.ControlData) + + if stat, err := os.Stat(debPath); err == nil { + p.Size = stat.Size() + } + + f, err := os.Open(debPath) + + if err != nil { + return nil, err + } + + var ( + md5hash = md5.New() + sha1hash = sha1.New() + sha256hash = sha256.New() + ) + + w := io.MultiWriter(md5hash, sha1hash, sha256hash) + + if _, err := io.Copy(w, f); err != nil { + return nil, err + } + + p.MD5Hash = hex.EncodeToString(md5hash.Sum(nil)) + p.SHA1Hash = hex.EncodeToString(sha1hash.Sum(nil)) + p.SHA256Hash = hex.EncodeToString(sha256hash.Sum(nil)) + + return p, nil +} + +func createPackagesCached(config Conf, distro, arch string, packages map[string]*PackageFile) error { + stdFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages")) + if err != nil { + return fmt.Errorf("failed to create packages: %s", err) + } + defer stdFile.Close() + + gzipFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages.gz")) + if err != nil { + return fmt.Errorf("failed to create packages.gz: %s", err) + } + defer gzipFile.Close() + + gzWriter := gzip.NewWriter(gzipFile) + defer gzWriter.Close() + + stdOut := bufio.NewWriter(stdFile) + + // loop through each directory + // run inspectPackage + + for _, p := range packages { + var packBuf bytes.Buffer + + packBuf.WriteString(p.ControlData) + + dir := filepath.Join("pool/main", distro, arch, p.Info.Package[0:1], p.Info.Package, p.Name) + + fmt.Fprintf(&packBuf, "Filename: %s\n", strings.Replace(dir, "\\", "/", -1)) + fmt.Fprintf(&packBuf, "Size: %d\n", p.Size) + + fmt.Fprintf(&packBuf, "MD5sum: %s\n", p.MD5Hash) + fmt.Fprintf(&packBuf, "SHA1: %s\n", p.SHA1Hash) + fmt.Fprintf(&packBuf, "SHA256: %s\n", p.SHA256Hash) + packBuf.WriteString("\n\n") + + stdOut.Write(packBuf.Bytes()) + gzWriter.Write(packBuf.Bytes()) + } + + stdOut.Flush() + + gzWriter.Flush() + + return nil +} \ No newline at end of file diff --git a/src/meow.tf/deb-simple/server.go b/src/meow.tf/deb-simple/server.go new file mode 100644 index 0000000..eaaeaee --- /dev/null +++ b/src/meow.tf/deb-simple/server.go @@ -0,0 +1,200 @@ +package simple + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "sync" + "strings" + "golang.org/x/crypto/openpgp" + "runtime" + "errors" +) + +var VERSION string = "1.1.0" + +type Conf struct { + ListenPort string `json:"listenPort"` + RootRepoPath string `json:"rootRepoPath"` + SupportArch []string `json:"supportedArch"` + DistroNames []string `json:"distroNames"` + PGPSecretKey string `json:"pgpSecretKey"` + PGPPassphrase string `json:"pgpPassphrase"` + EnableSSL bool `json:"enableSSL"` + SSLCert string `json:"SSLcert"` + SSLKey string `json:"SSLkey"` + Key string `json:"key"` +} + +func (c Conf) DistPath(distro string) string { + return filepath.Join(c.RootRepoPath, "dists", distro) +} + +func (c Conf) ArchPath(distro, arch string) string { + return filepath.Join(c.RootRepoPath, "dists", distro, "main/binary-"+arch) +} + +func (c Conf) PoolPath(distro, arch string) string { + return filepath.Join(c.RootRepoPath, "pool/main", distro, arch) +} + +func (c Conf) PoolPackagePath(distro, arch, name string) string { + name = packageName(name) + return filepath.Join(c.RootRepoPath, "pool/main", distro, arch, name[0:1], name) +} + +func packageName(name string) string { + if index := strings.Index(name, "_"); index != -1 { + name = name[:index] + } + return name +} + +type DeleteObj struct { + Filename string + DistroName string + Arch string +} + +var ( + mutex sync.RWMutex + configFile = flag.String("c", "conf.json", "config file location") + flagShowVersion = flag.Bool("version", false, "Show dnsconfig version") + parsedConfig = Conf{} + pgpEntity *openpgp.Entity +) + +func Start() { + flag.Parse() + + if *flagShowVersion { + fmt.Printf("deb-simple %s (%s)\n", VERSION, runtime.Version()) + os.Exit(0) + } + + file, err := ioutil.ReadFile(*configFile) + + if err != nil { + log.Fatalln("unable to read config file, exiting...") + } + + if err := json.Unmarshal(file, &parsedConfig); err != nil { + log.Fatalln("unable to marshal config file, exiting...") + } + + if err := createDirs(parsedConfig); err != nil { + log.Println(err) + log.Fatalln("error creating directory structure, exiting") + } + + if err := setupPgp(parsedConfig); err != nil { + log.Println(err) + log.Fatalln("error loading pgp key, exiting") + } + + log.Println("Indexing packages...") + + for _, dist := range parsedConfig.DistroNames { + distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)} + + for _, arch := range parsedConfig.SupportArch { + files, err := buildPackageList(parsedConfig, dist, arch) + + if err != nil { + log.Fatalln("Unable to load packages:", err) + } + + distro.Architectures[arch] = files + + log.Println("Generating packages file for", dist, arch) + createPackagesCached(parsedConfig, dist, arch, files) + } + + distros[dist] = distro + } + + http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(parsedConfig.RootRepoPath)))) + http.Handle("/upload", uploadHandler(parsedConfig)) + http.Handle("/delete", deleteHandler(parsedConfig)) + + if parsedConfig.EnableSSL { + log.Println("running with SSL enabled") + log.Fatalln(http.ListenAndServeTLS(":"+parsedConfig.ListenPort, parsedConfig.SSLCert, parsedConfig.SSLKey, nil)) + } else { + log.Println("running without SSL enabled") + log.Fatalln(http.ListenAndServe(":"+parsedConfig.ListenPort, nil)) + } +} + +func setupPgp(config Conf) error { + if config.PGPSecretKey == "" { + return nil + } + + secretKey, err := os.Open(config.PGPSecretKey) + if err != nil { + return fmt.Errorf("failed to open private key ring file: %s", err) + } + defer secretKey.Close() + + entitylist, err := openpgp.ReadKeyRing(secretKey) + + if err != nil { + return fmt.Errorf("failed to read key ring: %s", err) + } + + if len(entitylist) < 1 { + return errors.New("no keys in key ring") + } + + pgpEntity = entitylist[0] + + passphrase := []byte(config.PGPPassphrase) + + if err := pgpEntity.PrivateKey.Decrypt(passphrase); err != nil { + return err + } + + for _, subkey := range pgpEntity.Subkeys { + err := subkey.PrivateKey.Decrypt(passphrase) + + if err != nil { + return err + } + } + + return nil +} + +func createDirs(config Conf) error { + for _, distro := range config.DistroNames { + for _, arch := range config.SupportArch { + if _, err := os.Stat(config.ArchPath(distro, arch)); err != nil { + if os.IsNotExist(err) { + log.Printf("Directory for %s (%s) does not exist, creating", distro, arch) + if err := os.MkdirAll(config.ArchPath(distro, arch), 0755); err != nil { + return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err) + } + } else { + return fmt.Errorf("error inspecting %s (%s): %s", distro, arch, err) + } + } + if _, err := os.Stat(config.PoolPath(distro, arch)); err != nil { + if os.IsNotExist(err) { + log.Printf("Directory for %s (%s) does not exist, creating", distro, arch) + if err := os.MkdirAll(config.PoolPath(distro, arch), 0755); err != nil { + return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err) + } + } else { + return fmt.Errorf("error inspecting %s (%s): %s", distro, arch, err) + } + } + } + } + return nil +} From d3278c8394fa2ae77444ea06722a7a0593b3c54b Mon Sep 17 00:00:00 2001 From: Tyler Date: Sun, 11 Jun 2017 03:29:39 -0400 Subject: [PATCH 2/4] Generate release file too --- src/meow.tf/deb-simple/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/meow.tf/deb-simple/server.go b/src/meow.tf/deb-simple/server.go index eaaeaee..b426200 100644 --- a/src/meow.tf/deb-simple/server.go +++ b/src/meow.tf/deb-simple/server.go @@ -113,6 +113,7 @@ func Start() { log.Println("Generating packages file for", dist, arch) createPackagesCached(parsedConfig, dist, arch, files) + createRelease(parsedConfig, dist, arch) } distros[dist] = distro From 463c1751a92182fe71df71b9720048b5f9bbf9da Mon Sep 17 00:00:00 2001 From: Tyler Date: Sun, 11 Jun 2017 03:35:27 -0400 Subject: [PATCH 3/4] Remove more debug --- src/meow.tf/deb-simple/apt.go | 4 +--- src/meow.tf/deb-simple/packages.go | 2 -- src/meow.tf/deb-simple/server.go | 30 +++++++++++++++++------------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/meow.tf/deb-simple/apt.go b/src/meow.tf/deb-simple/apt.go index ca78344..99d1e92 100644 --- a/src/meow.tf/deb-simple/apt.go +++ b/src/meow.tf/deb-simple/apt.go @@ -119,8 +119,6 @@ func createRelease(config Conf, distro, arch string) error { for _, file := range dirList { filePath := filepath.Join(config.DistPath(distro), basePath, file.Name()) - log.Println("File path:", filePath) - fileLocalPath := filepath.Join(basePath, file.Name()) fileLocalPath = strings.Replace(fileLocalPath, "\\", "/", -1) @@ -128,7 +126,7 @@ func createRelease(config Conf, distro, arch string) error { f, err := os.Open(filePath) if err != nil { - log.Println("error opening source file: ", err) + return err } var size int64 = file.Size() diff --git a/src/meow.tf/deb-simple/packages.go b/src/meow.tf/deb-simple/packages.go index 81fc1dd..91d7afa 100644 --- a/src/meow.tf/deb-simple/packages.go +++ b/src/meow.tf/deb-simple/packages.go @@ -15,7 +15,6 @@ import ( "crypto/md5" "crypto/sha1" "crypto/sha256" - "log" "github.com/blang/semver" ) @@ -118,7 +117,6 @@ func scanRecursive(path string, m map[string]*PackageFile) error { } func newPackageFile(path, name string) (*PackageFile, error) { - log.Println("New package from", path, name) p := &PackageFile{ Name: name, Path: filepath.Join(path, name), diff --git a/src/meow.tf/deb-simple/server.go b/src/meow.tf/deb-simple/server.go index b426200..7d037e1 100644 --- a/src/meow.tf/deb-simple/server.go +++ b/src/meow.tf/deb-simple/server.go @@ -102,19 +102,7 @@ func Start() { for _, dist := range parsedConfig.DistroNames { distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)} - for _, arch := range parsedConfig.SupportArch { - files, err := buildPackageList(parsedConfig, dist, arch) - - if err != nil { - log.Fatalln("Unable to load packages:", err) - } - - distro.Architectures[arch] = files - - log.Println("Generating packages file for", dist, arch) - createPackagesCached(parsedConfig, dist, arch, files) - createRelease(parsedConfig, dist, arch) - } + go scanInitialPackages(parsedConfig, distro) distros[dist] = distro } @@ -132,6 +120,22 @@ func Start() { } } +func scanInitialPackages(config Conf, dist *Distro) { + for _, arch := range config.SupportArch { + files, err := buildPackageList(config, dist.Name, arch) + + if err != nil { + log.Fatalln("Unable to load packages:", err) + } + + dist.Architectures[arch] = files + + log.Println("Generating packages file for", dist.Name, arch) + createPackagesCached(config, dist.Name, arch, files) + createRelease(config, dist.Name, arch) + } +} + func setupPgp(config Conf) error { if config.PGPSecretKey == "" { return nil From 2fa86f850cb789bc5b154f989623adc5ee9d718b Mon Sep 17 00:00:00 2001 From: Tyler Date: Sun, 11 Jun 2017 03:52:32 -0400 Subject: [PATCH 4/4] Add semver parsing to stop overwriting of newer packages --- src/meow.tf/deb-simple/http.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/meow.tf/deb-simple/http.go b/src/meow.tf/deb-simple/http.go index 9db6e7d..d86f75f 100644 --- a/src/meow.tf/deb-simple/http.go +++ b/src/meow.tf/deb-simple/http.go @@ -8,6 +8,7 @@ import ( "log" "encoding/json" "fmt" + "github.com/blang/semver" ) func uploadHandler(config Conf) http.Handler { @@ -35,6 +36,14 @@ func uploadHandler(config Conf) http.Handler { return } + force := false + + forceStr := r.URL.Query().Get("force") + + if forceStr != "" && forceStr == "true" { + force = true + } + reader, err := r.MultipartReader() if err != nil { @@ -109,9 +118,22 @@ func uploadHandler(config Conf) http.Handler { } if p, exists := packages[f.Info.Package]; exists { + v1, err1 := semver.Parse(p.Info.Version) + v2, err2 := semver.Parse(f.Info.Version) + + if err1 == nil && err2 == nil && v1.Compare(v2) > 0 && !force { + // Don't replace newer package + httpErrorf(w, "version in old package is greater than new: %s, %s - override with \"force\"", p.Info.Version, f.Info.Version) + return + } + // Archive old file log.Println("Replacing", p.Name, "with", f.Name) - os.Remove(p.Path) + + if err := os.Remove(p.Path); err != nil { + httpErrorf(w, "Unable to remove old package: %s", err) + return + } } packages[f.Info.Package] = f