tyler
/
deb-simple
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move data into src, cached package structure

This commit is contained in:
Tyler 2017-06-11 03:18:59 -04:00
parent 0511a71fea
commit a93947ff81
11 changed files with 513 additions and 883 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,5 +1,5 @@
/deb-simple
/repo
/src
vendor
conf.json
*.pprof

16
.gitlab-ci.yml
View File

@ -1,16 +0,0 @@
image: knightswarm/gobuild:1.6
before_script:
- go get github.com/tools/godep
- export GOPATH=$(pwd)
stages:
- build
compile:
stage: build
script:
- export DEB_SIMPLE_VERSION=`grep "var VERSION" main.go | awk -F\" '{print $2}'`
- godep restore
- chmod +x ./ci/package.sh
- ./ci/package.sh

174
main.go
View File

@ -1,177 +1,7 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"sync"
"strings"
"golang.org/x/crypto/openpgp"
"runtime"
"errors"
)
var VERSION string = "1.1.0"
type Conf struct {
ListenPort string `json:"listenPort"`
RootRepoPath string `json:"rootRepoPath"`
SupportArch []string `json:"supportedArch"`
DistroNames []string `json:"distroNames"`
PGPSecretKey string `json:"pgpSecretKey"`
PGPPassphrase string `json:"pgpPassphrase"`
EnableSSL bool `json:"enableSSL"`
SSLCert string `json:"SSLcert"`
SSLKey string `json:"SSLkey"`
Key string `json:"key"`
}
func (c Conf) DistPath(distro string) string {
return filepath.Join(c.RootRepoPath, "dists", distro)
}
func (c Conf) ArchPath(distro, arch string) string {
return filepath.Join(c.RootRepoPath, "dists", distro, "main/binary-"+arch)
}
func (c Conf) PoolPath(distro, arch string) string {
return filepath.Join(c.RootRepoPath, "pool/main", distro, arch)
}
func (c Conf) PoolPackagePath(distro, arch, name string) string {
name = packageName(name)
return filepath.Join(c.RootRepoPath, "pool/main", distro, arch, name[0:1], name)
}
func packageName(name string) string {
if index := strings.Index(name, "_"); index != -1 {
name = name[:index]
}
return name
}
type DeleteObj struct {
Filename string
DistroName string
Arch string
}
var (
mutex sync.Mutex
configFile = flag.String("c", "conf.json", "config file location")
flagShowVersion = flag.Bool("version", false, "Show dnsconfig version")
parsedConfig = Conf{}
pgpEntity *openpgp.Entity
)
import "meow.tf/deb-simple"
func main() {
flag.Parse()
if *flagShowVersion {
fmt.Printf("deb-simple %s (%s)\n", VERSION, runtime.Version())
os.Exit(0)
}
file, err := ioutil.ReadFile(*configFile)
if err != nil {
log.Fatalln("unable to read config file, exiting...")
}
if err := json.Unmarshal(file, &parsedConfig); err != nil {
log.Fatalln("unable to marshal config file, exiting...")
}
if err := createDirs(parsedConfig); err != nil {
log.Println(err)
log.Fatalln("error creating directory structure, exiting")
}
if err := setupPgp(parsedConfig); err != nil {
log.Println(err)
log.Fatalln("error loading pgp key, exiting")
}
http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(parsedConfig.RootRepoPath))))
http.Handle("/upload", uploadHandler(parsedConfig))
http.Handle("/delete", deleteHandler(parsedConfig))
if parsedConfig.EnableSSL {
log.Println("running with SSL enabled")
log.Fatalln(http.ListenAndServeTLS(":"+parsedConfig.ListenPort, parsedConfig.SSLCert, parsedConfig.SSLKey, nil))
} else {
log.Println("running without SSL enabled")
log.Fatalln(http.ListenAndServe(":"+parsedConfig.ListenPort, nil))
}
}
func setupPgp(config Conf) error {
if config.PGPSecretKey == "" {
return nil
}
secretKey, err := os.Open(config.PGPSecretKey)
if err != nil {
return fmt.Errorf("failed to open private key ring file: %s", err)
}
defer secretKey.Close()
entitylist, err := openpgp.ReadKeyRing(secretKey)
if err != nil {
return fmt.Errorf("failed to read key ring: %s", err)
}
if len(entitylist) < 1 {
return errors.New("no keys in key ring")
}
pgpEntity = entitylist[0]
passphrase := []byte(config.PGPPassphrase)
if err := pgpEntity.PrivateKey.Decrypt(passphrase); err != nil {
return err
}
for _, subkey := range pgpEntity.Subkeys {
err := subkey.PrivateKey.Decrypt(passphrase)
if err != nil {
return err
}
}
return nil
}
func createDirs(config Conf) error {
for _, distro := range config.DistroNames {
for _, arch := range config.SupportArch {
if _, err := os.Stat(config.ArchPath(distro, arch)); err != nil {
if os.IsNotExist(err) {
log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)
if err := os.MkdirAll(config.ArchPath(distro, arch), 0755); err != nil {
return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err)
}
} else {
return fmt.Errorf("error inspecting %s (%s): %s", distro, arch, err)
}
}
if _, err := os.Stat(config.PoolPath(distro, arch)); err != nil {
if os.IsNotExist(err) {
log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)
if err := os.MkdirAll(config.PoolPath(distro, arch), 0755); err != nil {
return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err)
}
} else {
return fmt.Errorf("error inspecting %s (%s): %s", distro, arch, err)
}
}
}
}
return nil
simple.Start()
}

535
main_test.go
View File

@ -1,535 +0,0 @@
package main
import (
"bytes"
"compress/gzip"
"crypto/md5"
"encoding/hex"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"testing"
)
var goodOutput = `Package: vim-tiny
Source: vim
Version: 2:7.4.052-1ubuntu3
Architecture: amd64
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
Installed-Size: 931
Depends: vim-common (= 2:7.4.052-1ubuntu3), libacl1 (>= 2.2.51-8), libc6 (>= 2.15), libselinux1 (>= 1.32), libtinfo5
Suggests: indent
Provides: editor
Section: editors
Priority: important
Homepage: http://www.vim.org/
Description: Vi IMproved - enhanced vi editor - compact version
Vim is an almost compatible version of the UNIX editor Vi.
.
Many new features have been added: multi level undo, syntax
highlighting, command line history, on-line help, filename
completion, block operations, folding, Unicode support, etc.
.
This package contains a minimal version of vim compiled with no
GUI and a small subset of features in order to keep small the
package size. This package does not depend on the vim-runtime
package, but installing it you will get its additional benefits
(online documentation, plugins, ...).
Original-Maintainer: Debian Vim Maintainers <pkg-vim-maintainers@lists.alioth.debian.org>
`
var goodPkgGzOutput = `Package: vim-tiny
Source: vim
Version: 2:7.4.052-1ubuntu3
Architecture: amd64
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
Installed-Size: 931
Depends: vim-common (= 2:7.4.052-1ubuntu3), libacl1 (>= 2.2.51-8), libc6 (>= 2.15), libselinux1 (>= 1.32), libtinfo5
Suggests: indent
Provides: editor
Section: editors
Priority: important
Homepage: http://www.vim.org/
Description: Vi IMproved - enhanced vi editor - compact version
Vim is an almost compatible version of the UNIX editor Vi.
.
Many new features have been added: multi level undo, syntax
highlighting, command line history, on-line help, filename
completion, block operations, folding, Unicode support, etc.
.
This package contains a minimal version of vim compiled with no
GUI and a small subset of features in order to keep small the
package size. This package does not depend on the vim-runtime
package, but installing it you will get its additional benefits
(online documentation, plugins, ...).
Original-Maintainer: Debian Vim Maintainers <pkg-vim-maintainers@lists.alioth.debian.org>
Filename: dists/stable/main/binary-cats/test.deb
Size: 391240
MD5sum: 0ec79417129746ff789fcff0976730c5
SHA1: b2ac976af80f0f50a8336402d5a29c67a2880b9b
SHA256: 9938ec82a8c882ebc2d59b64b0bf2ac01e9cbc5a235be4aa268d4f8484e75eab
`
var goodPkgGzOutputNonDefault = `Package: vim-tiny
Source: vim
Version: 2:7.4.052-1ubuntu3
Architecture: amd64
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
Installed-Size: 931
Depends: vim-common (= 2:7.4.052-1ubuntu3), libacl1 (>= 2.2.51-8), libc6 (>= 2.15), libselinux1 (>= 1.32), libtinfo5
Suggests: indent
Provides: editor
Section: editors
Priority: important
Homepage: http://www.vim.org/
Description: Vi IMproved - enhanced vi editor - compact version
Vim is an almost compatible version of the UNIX editor Vi.
.
Many new features have been added: multi level undo, syntax
highlighting, command line history, on-line help, filename
completion, block operations, folding, Unicode support, etc.
.
This package contains a minimal version of vim compiled with no
GUI and a small subset of features in order to keep small the
package size. This package does not depend on the vim-runtime
package, but installing it you will get its additional benefits
(online documentation, plugins, ...).
Original-Maintainer: Debian Vim Maintainers <pkg-vim-maintainers@lists.alioth.debian.org>
Filename: dists/blah/main/binary-cats/test.deb
Size: 391240
MD5sum: 0ec79417129746ff789fcff0976730c5
SHA1: b2ac976af80f0f50a8336402d5a29c67a2880b9b
SHA256: 9938ec82a8c882ebc2d59b64b0bf2ac01e9cbc5a235be4aa268d4f8484e75eab
`
func TestCreateDirs(t *testing.T) {
pwd, err := os.Getwd()
if err != nil {
t.Errorf("Unable to get current working directory: %s", err)
}
config := Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"stable"}, EnableSSL: false}
// sanity check...
if config.RootRepoPath != pwd+"/testing" {
t.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing")
}
t.Log("creating temp dirs in ", config.RootRepoPath)
if err := createDirs(config); err != nil {
t.Errorf("createDirs() failed ")
}
for _, archDir := range config.SupportArch {
if _, err := os.Stat(config.RootRepoPath + "/dists/stable/main/binary-" + archDir); err != nil {
if os.IsNotExist(err) {
t.Errorf("Directory for %s does not exist", archDir)
}
}
}
// cleanup
if err := os.RemoveAll(config.RootRepoPath); err != nil {
t.Errorf("error cleaning up after createDirs(): %s", err)
}
// create temp file
tempFile, err := os.Create(pwd + "/tempFile")
if err != nil {
t.Fatalf("create %s: %s", pwd+"/tempFile", err)
}
defer tempFile.Close()
config.RootRepoPath = pwd + "/tempFile"
// Can't make directory named after file.
if err := createDirs(config); err == nil {
t.Errorf("createDirs() should have failed but did not")
}
// cleanup
if err := os.RemoveAll(pwd + "/tempFile"); err != nil {
t.Errorf("error cleaning up after createDirs(): %s", err)
}
}
func TestInspectPackage(t *testing.T) {
parsedControl, err := inspectPackage("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb")
if err != nil {
t.Error("inspectPackage() error: %s", err)
}
if parsedControl != goodOutput {
t.Errorf("control file does not match")
}
_, err = inspectPackage("thisfileshouldnotexist")
if err == nil {
t.Error("inspectPackage() should have failed, it did not")
}
}
func TestInspectPackageControl(t *testing.T) {
sampleDeb, err := ioutil.ReadFile("samples/control.tar.gz")
if err != nil {
t.Errorf("error opening sample deb file: %s", err)
}
var controlBuf bytes.Buffer
cfReader := bytes.NewReader(sampleDeb)
io.Copy(&controlBuf, cfReader)
parsedControl, err := inspectPackageControl(controlBuf)
if err != nil {
t.Error("error inspecting control file: %s", err)
}
if parsedControl != goodOutput {
t.Errorf("control file does not match")
}
var failControlBuf bytes.Buffer
_, err = inspectPackageControl(failControlBuf)
if err == nil {
t.Error("inspectPackageControl() should have failed, it did not")
}
}
func TestCreatePackagesGz(t *testing.T) {
pwd, err := os.Getwd()
if err != nil {
t.Errorf("Unable to get current working directory: %s", err)
}
config := Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"stable"}, EnableSSL: false}
// sanity check...
if config.RootRepoPath != pwd+"/testing" {
t.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing")
}
// copy sample deb to repo location (assuming it exists)
origDeb, err := os.Open("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb")
if err != nil {
t.Errorf("error opening up sample deb: %s", err)
}
defer origDeb.Close()
for _, archDir := range config.SupportArch {
// do not use the built-in createDirs() in case it is broken
if err := os.MkdirAll(config.RootRepoPath+"/dists/stable/main/binary-"+archDir, 0755); err != nil {
t.Errorf("error creating directory for %s: %s\n", archDir, err)
}
copyDeb, err := os.Create(config.RootRepoPath + "/dists/stable/main/binary-" + archDir + "/test.deb")
if err != nil {
t.Errorf("error creating copy of deb: %s", err)
}
_, err = io.Copy(copyDeb, origDeb)
if err != nil {
t.Errorf("error writing copy of deb: %s", err)
}
if err := copyDeb.Close(); err != nil {
t.Errorf("error saving copy of deb: %s", err)
}
}
if err := createPackagesGz(config, "stable", "cats"); err != nil {
t.Errorf("error creating packages gzip for cats")
}
pkgGzip, err := ioutil.ReadFile(config.RootRepoPath + "/dists/stable/main/binary-cats/Packages.gz")
if err != nil {
t.Errorf("error reading Packages.gz: %s", err)
}
pkgReader, err := gzip.NewReader(bytes.NewReader(pkgGzip))
if err != nil {
t.Errorf("error reading existing Packages.gz: %s", err)
}
buf := bytes.NewBuffer(nil)
io.Copy(buf, pkgReader)
if goodPkgGzOutput != string(buf.Bytes()) {
t.Errorf("Packages.gz does not match, returned value is: %s", string(buf.Bytes()))
}
// cleanup
if err := os.RemoveAll(config.RootRepoPath); err != nil {
t.Errorf("error cleaning up after createPackagesGz(): %s", err)
}
// create temp file
tempFile, err := os.Create(pwd + "/tempFile")
if err != nil {
t.Fatalf("create %s: %s", pwd+"/tempFile", err)
}
defer tempFile.Close()
config.RootRepoPath = pwd + "/tempFile"
// Can't make directory named after file
if err := createPackagesGz(config, "stable", "cats"); err == nil {
t.Errorf("createPackagesGz() should have failed, it did not")
}
// cleanup
if err := os.RemoveAll(pwd + "/tempFile"); err != nil {
t.Errorf("error cleaning up after createDirs(): %s", err)
}
}
func TestCreatePackagesGzNonDefault(t *testing.T) {
pwd, err := os.Getwd()
if err != nil {
t.Errorf("Unable to get current working directory: %s", err)
}
config := Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"blah"}, EnableSSL: false}
// sanity check...
if config.RootRepoPath != pwd+"/testing" {
t.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing")
}
// copy sample deb to repo location (assuming it exists)
origDeb, err := os.Open("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb")
if err != nil {
t.Errorf("error opening up sample deb: %s", err)
}
defer origDeb.Close()
for _, archDir := range config.SupportArch {
// do not use the built-in createDirs() in case it is broken
if err := os.MkdirAll(config.RootRepoPath+"/dists/blah/main/binary-"+archDir, 0755); err != nil {
t.Errorf("error creating directory for %s: %s\n", archDir, err)
}
copyDeb, err := os.Create(config.RootRepoPath + "/dists/blah/main/binary-" + archDir + "/test.deb")
if err != nil {
t.Errorf("error creating copy of deb: %s", err)
}
_, err = io.Copy(copyDeb, origDeb)
if err != nil {
t.Errorf("error writing copy of deb: %s", err)
}
if err := copyDeb.Close(); err != nil {
t.Errorf("error saving copy of deb: %s", err)
}
}
if err := createPackagesGz(config, "blah", "cats"); err != nil {
t.Errorf("error creating packages gzip for cats")
}
pkgGzip, err := ioutil.ReadFile(config.RootRepoPath + "/dists/blah/main/binary-cats/Packages.gz")
if err != nil {
t.Errorf("error reading Packages.gz: %s", err)
}
pkgReader, err := gzip.NewReader(bytes.NewReader(pkgGzip))
if err != nil {
t.Errorf("error reading existing Packages.gz: %s", err)
}
buf := bytes.NewBuffer(nil)
io.Copy(buf, pkgReader)
if goodPkgGzOutputNonDefault != string(buf.Bytes()) {
t.Errorf("Packages.gz does not match, returned value is: %s", string(buf.Bytes()))
}
// cleanup
if err := os.RemoveAll(config.RootRepoPath); err != nil {
t.Errorf("error cleaning up after createPackagesGz(): %s", err)
}
}
func TestUploadHandler(t *testing.T) {
pwd, err := os.Getwd()
if err != nil {
t.Errorf("Unable to get current working directory: %s", err)
}
config := Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"stable"}, EnableSSL: false}
// sanity check...
if config.RootRepoPath != pwd+"/testing" {
t.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing")
}
uploadHandle := uploadHandler(config)
// GET
req, _ := http.NewRequest("GET", "", nil)
w := httptest.NewRecorder()
uploadHandle.ServeHTTP(w, req)
if w.Code != http.StatusMethodNotAllowed {
t.Errorf("uploadHandler GET returned %v, should be %v", w.Code, http.StatusMethodNotAllowed)
}
// POST
// create "all" arch as it's the default
if err := os.MkdirAll(config.RootRepoPath+"/dists/stable/main/binary-all", 0755); err != nil {
t.Error("error creating directory for POST testing: %s", err)
}
sampleDeb, err := os.Open("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb")
if err != nil {
t.Errorf("error opening sample deb file: %s", err)
}
defer sampleDeb.Close()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "vim-tiny_7.4.052-1ubuntu3_amd64.deb")
if err != nil {
t.Errorf("error FormFile: %s", err)
}
_, err = io.Copy(part, sampleDeb)
if err != nil {
t.Errorf("error copying sampleDeb to FormFile: %s", err)
}
if err := writer.Close(); err != nil {
t.Errorf("error closing form writer: %s", err)
}
req, _ = http.NewRequest("POST", "", body)
req.Header.Add("Content-Type", writer.FormDataContentType())
w = httptest.NewRecorder()
uploadHandle.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Errorf("uploadHandler POST returned %v, should be %v", w.Code, http.StatusOK)
}
// verify uploaded file matches sample
uploadFile, _ := ioutil.ReadFile(config.RootRepoPath + "/dists/stable/main/binary-all/vim-tiny_7.4.052-1ubuntu3_amd64.deb")
uploadmd5hash := md5.New()
uploadmd5hash.Write(uploadFile)
uploadFilemd5 := hex.EncodeToString(uploadmd5hash.Sum(nil))
sampleFile, _ := ioutil.ReadFile("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb")
samplemd5hash := md5.New()
samplemd5hash.Write(sampleFile)
sampleFilemd5 := hex.EncodeToString(samplemd5hash.Sum(nil))
if uploadFilemd5 != sampleFilemd5 {
t.Errorf("uploaded file MD5 is %s, should be %s", uploadFilemd5, sampleFilemd5)
}
// cleanup
if err := os.RemoveAll(config.RootRepoPath); err != nil {
t.Errorf("error cleaning up after uploadHandler(): %s", err)
}
// create temp file
tempFile, err := os.Create(pwd + "/tempFile")
if err != nil {
t.Fatalf("create %s: %s", pwd+"/tempFile", err)
}
defer tempFile.Close()
config.RootRepoPath = pwd + "/tempFile"
// Can't make directory named after file
uploadHandle = uploadHandler(config)
failBody := &bytes.Buffer{}
failWriter := multipart.NewWriter(failBody)
failPart, err := failWriter.CreateFormFile("file", "vim-tiny_7.4.052-1ubuntu3_amd64.deb")
if err != nil {
t.Errorf("error FormFile: %s", err)
}
_, err = io.Copy(failPart, sampleDeb)
if err != nil {
t.Errorf("error copying sampleDeb to FormFile: %s", err)
}
if err := failWriter.Close(); err != nil {
t.Errorf("error closing form writer: %s", err)
}
req, _ = http.NewRequest("POST", "", failBody)
req.Header.Add("Content-Type", failWriter.FormDataContentType())
w = httptest.NewRecorder()
uploadHandle.ServeHTTP(w, req)
if w.Code != http.StatusInternalServerError {
t.Errorf("uploadHandler POST returned %v, should be %v", w.Code, http.StatusInternalServerError)
}
// cleanup
if err := os.RemoveAll(pwd + "/tempFile"); err != nil {
t.Errorf("error cleaning up after createDirs(): %s", err)
}
}
func TestDeleteHandler(t *testing.T) {
pwd, err := os.Getwd()
if err != nil {
t.Errorf("Unable to get current working directory: %s", err)
}
config := Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"stable"}, EnableSSL: false}
// sanity check...
if config.RootRepoPath != pwd+"/testing" {
t.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing")
}
deleteHandle := deleteHandler(config)
// GET
req, _ := http.NewRequest("GET", "", nil)
w := httptest.NewRecorder()
deleteHandle.ServeHTTP(w, req)
if w.Code != http.StatusMethodNotAllowed {
t.Errorf("deleteHandler GET returned %v, should be %v", w.Code, http.StatusMethodNotAllowed)
}
// DELETE
// create "all" arch as it's the default
if err := os.MkdirAll(config.RootRepoPath+"/dists/stable/main/binary-all", 0755); err != nil {
t.Error("error creating directory for POST testing: %s", err)
}
tempDeb, err := os.Create(config.RootRepoPath + "/dists/stable/main/binary-all/myapp.deb")
if err != nil {
t.Fatalf("create %s: %s", config.RootRepoPath+"/dists/stable/main/binary-all/myapp.deb", err)
}
defer tempDeb.Close()
req, _ = http.NewRequest("DELETE", "", bytes.NewBufferString("{\"filename\":\"myapp.deb\",\"arch\":\"all\", \"distroName\":\"stable\"}"))
w = httptest.NewRecorder()
deleteHandle.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Errorf("deleteHandler DELETE returned %v, should be %v", w.Code, http.StatusOK)
}
// cleanup
if err := os.RemoveAll(config.RootRepoPath); err != nil {
t.Errorf("error cleaning up after uploadHandler(): %s", err)
}
// create temp file
tempFile, err := os.Create(pwd + "/tempFile")
if err != nil {
t.Fatalf("create %s: %s", pwd+"/tempFile", err)
}
defer tempFile.Close()
config.RootRepoPath = pwd + "/tempFile"
// Can't make directory named after file
deleteHandle = deleteHandler(config)
req, _ = http.NewRequest("DELETE", "", bytes.NewBufferString("{\"filename\":\"myapp.deb\",\"arch\":\"amd64\", \"distroName\":\"stable\"}"))
w = httptest.NewRecorder()
deleteHandle.ServeHTTP(w, req)
if w.Code != http.StatusInternalServerError {
t.Errorf("deleteHandler DELETE returned %v, should be %v", w.Code, http.StatusInternalServerError)
}
// cleanup
if err := os.RemoveAll(pwd + "/tempFile"); err != nil {
t.Errorf("error cleaning up after createDirs(): %s", err)
}
}
func BenchmarkUploadHandler(b *testing.B) {
pwd, err := os.Getwd()
if err != nil {
b.Errorf("Unable to get current working directory: %s", err)
}
config := &Conf{ListenPort: "9666", RootRepoPath: pwd + "/testing", SupportArch: []string{"cats", "dogs"}, DistroNames: []string{"stable"}, EnableSSL: false}
// sanity check...
if config.RootRepoPath != pwd+"/testing" {
b.Errorf("RootRepoPath is %s, should be %s\n ", config.RootRepoPath, pwd+"/testing")
}
uploadHandle := uploadHandler(*config)
if err := os.MkdirAll(config.RootRepoPath+"/dists/stable/main/binary-all", 0755); err != nil {
b.Errorf("error creating directory for POST testing: %s", err)
}
sampleDeb, err := os.Open("samples/vim-tiny_7.4.052-1ubuntu3_amd64.deb")
if err != nil {
b.Errorf("error opening sample deb file: %s", err)
}
defer sampleDeb.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
// temporary (i hope) hack to solve "http: MultipartReader called twice" error
b.StopTimer()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "vim-tiny_7.4.052-1ubuntu3_amd64.deb")
if err != nil {
b.Errorf("error FormFile: %s", err)
}
if _, err := io.Copy(part, sampleDeb); err != nil {
b.Errorf("error copying sampleDeb to FormFile: %s", err)
}
if err := writer.Close(); err != nil {
b.Errorf("error closing form writer: %s", err)
}
req, _ := http.NewRequest("POST", "/upload?distro=stable", body)
req.Header.Add("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
b.StartTimer()
uploadHandle.ServeHTTP(w, req)
}
b.StopTimer()
// cleanup
_ = os.RemoveAll(config.RootRepoPath)
}

31
packages.go
View File

@ -1,31 +0,0 @@
package main
import (
"regexp"
"log"
"reflect"
)
var (
variableRegexp = regexp.MustCompile("(.*?):\\s*(.*)")
)
type Package struct {
Package string
Version string
}
func parsePackageData(ctlData string) (*Package, error) {
res := &Package{}
for _, match := range variableRegexp.FindAllStringSubmatch(ctlData, -1) {
switch match[1] {
case "Package":
res.Package = match[2]
case "Version":
res.Version = match[2]
}
}
return res, nil
}

118
apt.go → src/meow.tf/deb-simple/apt.go
View File

@ -1,4 +1,4 @@
package main
package simple
import (
"os"
@ -17,7 +17,6 @@ import (
"crypto/sha256"
"encoding/hex"
"golang.org/x/crypto/openpgp"
"bufio"
"time"
)
@ -89,125 +88,14 @@ func inspectPackageControl(filename bytes.Buffer) (string, error) {
return "", nil
}
func createPackagesGz(config Conf, distro, arch string) error {
stdFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages"))
if err != nil {
return fmt.Errorf("failed to create packages: %s", err)
}
defer stdFile.Close()
gzipFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages.gz"))
if err != nil {
return fmt.Errorf("failed to create packages.gz: %s", err)
}
defer gzipFile.Close()
gzWriter := gzip.NewWriter(gzipFile)
defer gzWriter.Close()
stdOut := bufio.NewWriter(stdFile)
// loop through each directory
// run inspectPackage
poolPath := config.PoolPath(distro, arch)
dirList, err := ioutil.ReadDir(poolPath)
if err != nil {
return fmt.Errorf("scanning: %s: %s", poolPath, err)
}
for _, firstChar := range dirList {
// Recursing isn't fun...
dirList, err := ioutil.ReadDir(filepath.Join(poolPath, firstChar.Name()))
if err != nil {
return fmt.Errorf("scanning: %s: %s", poolPath, err)
}
for _, p := range dirList {
dirList, err := ioutil.ReadDir(filepath.Join(poolPath, firstChar.Name(), p.Name()))
if err != nil {
return fmt.Errorf("scanning: %s: %s", poolPath, err)
}
for _, debFile := range dirList {
if strings.HasSuffix(debFile.Name(), "deb") {
var packBuf bytes.Buffer
debName := packageName(debFile.Name())
debPath := filepath.Join(config.PoolPackagePath(distro, arch, debFile.Name()), debFile.Name())
tempCtlData, err := inspectPackage(debPath)
if err != nil {
return err
}
packBuf.WriteString(tempCtlData)
var size int64 = debFile.Size()
if size == 0 {
if stat, err := os.Stat(debPath); err == nil {
size = stat.Size()
}
}
dir := filepath.Join("pool/main", distro, arch, debName[0:1], debName, debFile.Name())
log.Println("path:", dir)
fmt.Fprintf(&packBuf, "Filename: %s\n", strings.Replace(dir, "\\", "/", -1))
fmt.Fprintf(&packBuf, "Size: %d\n", size)
f, err := os.Open(debPath)
if err != nil {
log.Println("error opening deb file: ", err)
}
var (
md5hash = md5.New()
sha1hash = sha1.New()
sha256hash = sha256.New()
)
f.Seek(0, 0)
if _, err := io.Copy(md5hash, f); err != nil {
log.Println("error with the md5 hash: ", err)
}
fmt.Fprintf(&packBuf, "MD5sum: %s\n",
hex.EncodeToString(md5hash.Sum(nil)))
f.Seek(0, 0)
if _, err = io.Copy(sha1hash, f); err != nil {
log.Println("error with the sha1 hash: ", err)
}
fmt.Fprintf(&packBuf, "SHA1: %s\n",
hex.EncodeToString(sha1hash.Sum(nil)))
f.Seek(0, 0)
if _, err = io.Copy(sha256hash, f); err != nil {
log.Println("error with the sha256 hash: ", err)
}
fmt.Fprintf(&packBuf, "SHA256: %s\n",
hex.EncodeToString(sha256hash.Sum(nil)))
packBuf.WriteString("\n\n")
stdOut.Write(packBuf.Bytes())
gzWriter.Write(packBuf.Bytes())
f.Close()
f = nil
}
}
}
}
stdOut.Flush()
gzWriter.Flush()
return nil
}
func createRelease(config Conf, distro, arch string) error {
outfile, err := os.Create(filepath.Join(config.DistPath(distro), "Release"))
if err != nil {
return fmt.Errorf("failed to create Release: %s", err)
}
defer outfile.Close()
var packBuf bytes.Buffer

8
glide.lock → src/meow.tf/deb-simple/glide.lock generated
View File

@ -1,10 +1,12 @@
hash: f49345e0dcd6cb4f3c49af64a95afbea2e608503cd6c1909d300e4a457ef65a6
updated: 2017-05-02T21:07:03.8312824-04:00
hash: 79d0f0e99e99ce9318bb0e8c6ab398657a20679dbe3c421c6fc588db7a738131
updated: 2017-06-11T03:04:15.102617-04:00
imports:
- name: github.com/blakesmith/ar
version: 8bd4349a67f2533b078dbc524689d15dba0f4659
- name: github.com/blang/semver
version: b38d23b8782a487059e8fc8773e9a5b228a77cb6
- name: golang.org/x/crypto
version: d1464577745bc7f4e74f65be9cfbd09436a729d6
version: 7e9105388ebff089b3f99f0ef676ea55a6da3a7e
subpackages:
- cast5
- openpgp

2
glide.yaml → src/meow.tf/deb-simple/glide.yaml
View File

@ -4,3 +4,5 @@ import:
- package: golang.org/x/crypto
subpackages:
- openpgp
- package: github.com/blang/semver
version: ^3.5.0

95
http.go → src/meow.tf/deb-simple/http.go
View File

@ -1,4 +1,4 @@
package main
package simple
import (
"net/http"
@ -17,28 +17,62 @@ func uploadHandler(config Conf) http.Handler {
return
}
archType := r.URL.Query().Get("arch")
if archType == "" {
archType = "all"
}
distroName := r.URL.Query().Get("distro")
if distroName == "" {
distroName = "stable"
}
key := r.URL.Query().Get("key")
if key == "" || key != config.Key {
http.Error(w, "unauthorized", 403)
return
}
reader, err := r.MultipartReader()
if err != nil {
httpErrorf(w, "error creating multipart reader: %s", err)
return
}
mutex.RLock()
distro, exists := distros[distroName]
if !exists {
httpErrorf(w, "invalid distro: %s", distroName)
mutex.RUnlock()
return
}
packages, exists := distro.Architectures[archType]
if !exists {
httpErrorf(w, "invalid arch: %s", archType)
mutex.RUnlock()
return
}
mutex.RUnlock()
// Lock to prevent concurrent modification
mutex.Lock()
defer mutex.Unlock()
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if part.FileName() == "" {
continue
}
@ -54,22 +88,40 @@ func uploadHandler(config Conf) http.Handler {
}
dst, err := os.Create(filepath.Join(newPath, part.FileName()))
if err != nil {
httpErrorf(w, "error creating deb file: %s", err)
return
}
defer dst.Close()
if _, err := io.Copy(dst, part); err != nil {
httpErrorf(w, "error writing deb file: %s", err)
return
}
// Get package name, if it already exists remove the old file.
f, err := newPackageFile(newPath, part.FileName())
if err != nil {
httpErrorf(w, "error loading package info: %s", err)
}
if p, exists := packages[f.Info.Package]; exists {
// Archive old file
log.Println("Replacing", p.Name, "with", f.Name)
os.Remove(p.Path)
}
packages[f.Info.Package] = f
}
mutex.Lock()
defer mutex.Unlock()
log.Println("got lock, updating package list...")
if err := createPackagesGz(config, distroName, archType); err != nil {
// Recreate the package index and release file.
if err := createPackagesCached(config, distroName, archType, packages); err != nil {
httpErrorf(w, "error creating package: %s", err)
return
}
@ -91,17 +143,42 @@ func deleteHandler(config Conf) http.Handler {
http.Error(w, "method not supported", http.StatusMethodNotAllowed)
return
}
var toDelete DeleteObj
if err := json.NewDecoder(r.Body).Decode(&toDelete); err != nil {
var req DeleteObj
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
httpErrorf(w, "failed to decode json: %s", err)
return
}
debPath := filepath.Join(config.ArchPath(toDelete.DistroName, toDelete.Arch), toDelete.Filename)
mutex.RLock()
distro, exists := distros[req.DistroName]
if !exists {
httpErrorf(w, "invalid distro: %s", req.DistroName)
mutex.RUnlock()
return
}
packages, exists := distro.Architectures[req.Arch]
if !exists {
httpErrorf(w, "invalid arch: %s", req.Arch)
mutex.RUnlock()
return
}
mutex.RUnlock()
debPath := filepath.Join(config.ArchPath(req.DistroName, req.Arch), req.Filename)
if err := os.Remove(debPath); err != nil {
httpErrorf(w, "failed to delete: %s", err)
return
}
key := r.URL.Query().Get("key")
if key == "" || key != config.Key {
http.Error(w, "unauthorized", http.StatusForbidden)
return
@ -110,12 +187,12 @@ func deleteHandler(config Conf) http.Handler {
defer mutex.Unlock()
log.Println("got lock, updating package list...")
if err := createPackagesGz(config, toDelete.DistroName, toDelete.Arch); err != nil {
if err := createPackagesCached(config, req.DistroName, req.Arch, packages); err != nil {
httpErrorf(w, "failed to delete package: %s", err)
return
}
if err := createRelease(config, toDelete.DistroName, toDelete.Arch); err != nil {
if err := createRelease(config, req.DistroName, req.Arch); err != nil {
httpErrorf(w, "failed to delete package: %s", err)
return
}

213
src/meow.tf/deb-simple/packages.go Normal file
View File

@ -0,0 +1,213 @@
package simple
import (
"regexp"
"fmt"
"path/filepath"
"strings"
"os"
"bytes"
"bufio"
"compress/gzip"
"encoding/hex"
"io"
"io/ioutil"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"log"
"github.com/blang/semver"
)
var (
variableRegexp = regexp.MustCompile("(.*?):\\s*(.*)")
)
type PackageFile struct {
Path string
Name string
Size int64
MD5Hash string
SHA1Hash string
SHA256Hash string
ControlData string
Info *Package
}
type Package struct {
Package string
Version string
Architecture string
}
func parsePackageData(ctlData string) *Package {
res := &Package{}
for _, match := range variableRegexp.FindAllStringSubmatch(ctlData, -1) {
switch match[1] {
case "Package":
res.Package = match[2]
case "Architecture":
res.Architecture = match[2]
case "Version":
res.Version = match[2]
}
}
return res
}
type Distro struct {
Name string
Architectures map[string]map[string]*PackageFile
}
var (
distros map[string]*Distro = make(map[string]*Distro)
)
func buildPackageList(config Conf, distro, arch string) (map[string]*PackageFile, error) {
m := make(map[string]*PackageFile)
if err := scanRecursive(config.PoolPath(distro, arch), m); err != nil {
return nil, err
}
return m, nil
}
func scanRecursive(path string, m map[string]*PackageFile) error {
dirList, err := ioutil.ReadDir(path)
if err != nil {
return err
}
for _, file := range dirList {
if file.IsDir() {
if err := scanRecursive(filepath.Join(path, file.Name()), m); err != nil {
return err
}
continue
}
if !strings.HasSuffix(file.Name(), "deb") {
continue
}
p, err := newPackageFile(path, file.Name())
if err != nil {
return err
}
if old, exists := m[p.Info.Package]; exists {
v1, err := semver.Parse(old.Info.Version)
v2, err2 := semver.Parse(p.Info.Version)
if err == nil && err2 == nil && v1.Compare(v2) > 0 {
// Use old version
continue
}
}
m[p.Info.Package] = p
}
return nil
}
func newPackageFile(path, name string) (*PackageFile, error) {
log.Println("New package from", path, name)
p := &PackageFile{
Name: name,
Path: filepath.Join(path, name),
}
debPath := filepath.Join(path, name)
var err error
p.ControlData, err = inspectPackage(debPath)
if err != nil {
return nil, err
}
p.Info = parsePackageData(p.ControlData)
if stat, err := os.Stat(debPath); err == nil {
p.Size = stat.Size()
}
f, err := os.Open(debPath)
if err != nil {
return nil, err
}
var (
md5hash = md5.New()
sha1hash = sha1.New()
sha256hash = sha256.New()
)
w := io.MultiWriter(md5hash, sha1hash, sha256hash)
if _, err := io.Copy(w, f); err != nil {
return nil, err
}
p.MD5Hash = hex.EncodeToString(md5hash.Sum(nil))
p.SHA1Hash = hex.EncodeToString(sha1hash.Sum(nil))
p.SHA256Hash = hex.EncodeToString(sha256hash.Sum(nil))
return p, nil
}
func createPackagesCached(config Conf, distro, arch string, packages map[string]*PackageFile) error {
stdFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages"))
if err != nil {
return fmt.Errorf("failed to create packages: %s", err)
}
defer stdFile.Close()
gzipFile, err := os.Create(filepath.Join(config.ArchPath(distro, arch), "Packages.gz"))
if err != nil {
return fmt.Errorf("failed to create packages.gz: %s", err)
}
defer gzipFile.Close()
gzWriter := gzip.NewWriter(gzipFile)
defer gzWriter.Close()
stdOut := bufio.NewWriter(stdFile)
// loop through each directory
// run inspectPackage
for _, p := range packages {
var packBuf bytes.Buffer
packBuf.WriteString(p.ControlData)
dir := filepath.Join("pool/main", distro, arch, p.Info.Package[0:1], p.Info.Package, p.Name)
fmt.Fprintf(&packBuf, "Filename: %s\n", strings.Replace(dir, "\\", "/", -1))
fmt.Fprintf(&packBuf, "Size: %d\n", p.Size)
fmt.Fprintf(&packBuf, "MD5sum: %s\n", p.MD5Hash)
fmt.Fprintf(&packBuf, "SHA1: %s\n", p.SHA1Hash)
fmt.Fprintf(&packBuf, "SHA256: %s\n", p.SHA256Hash)
packBuf.WriteString("\n\n")
stdOut.Write(packBuf.Bytes())
gzWriter.Write(packBuf.Bytes())
}
stdOut.Flush()
gzWriter.Flush()
return nil
}

200
src/meow.tf/deb-simple/server.go Normal file
View File

@ -0,0 +1,200 @@
package simple
import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"sync"
"strings"
"golang.org/x/crypto/openpgp"
"runtime"
"errors"
)
var VERSION string = "1.1.0"
type Conf struct {
ListenPort string `json:"listenPort"`
RootRepoPath string `json:"rootRepoPath"`
SupportArch []string `json:"supportedArch"`
DistroNames []string `json:"distroNames"`
PGPSecretKey string `json:"pgpSecretKey"`
PGPPassphrase string `json:"pgpPassphrase"`
EnableSSL bool `json:"enableSSL"`
SSLCert string `json:"SSLcert"`
SSLKey string `json:"SSLkey"`
Key string `json:"key"`
}
func (c Conf) DistPath(distro string) string {
return filepath.Join(c.RootRepoPath, "dists", distro)
}
func (c Conf) ArchPath(distro, arch string) string {
return filepath.Join(c.RootRepoPath, "dists", distro, "main/binary-"+arch)
}
func (c Conf) PoolPath(distro, arch string) string {
return filepath.Join(c.RootRepoPath, "pool/main", distro, arch)
}
func (c Conf) PoolPackagePath(distro, arch, name string) string {
name = packageName(name)
return filepath.Join(c.RootRepoPath, "pool/main", distro, arch, name[0:1], name)
}
func packageName(name string) string {
if index := strings.Index(name, "_"); index != -1 {
name = name[:index]
}
return name
}
type DeleteObj struct {
Filename string
DistroName string
Arch string
}
var (
mutex sync.RWMutex
configFile = flag.String("c", "conf.json", "config file location")
flagShowVersion = flag.Bool("version", false, "Show dnsconfig version")
parsedConfig = Conf{}
pgpEntity *openpgp.Entity
)
func Start() {
flag.Parse()
if *flagShowVersion {
fmt.Printf("deb-simple %s (%s)\n", VERSION, runtime.Version())
os.Exit(0)
}
file, err := ioutil.ReadFile(*configFile)
if err != nil {
log.Fatalln("unable to read config file, exiting...")
}
if err := json.Unmarshal(file, &parsedConfig); err != nil {
log.Fatalln("unable to marshal config file, exiting...")
}
if err := createDirs(parsedConfig); err != nil {
log.Println(err)
log.Fatalln("error creating directory structure, exiting")
}
if err := setupPgp(parsedConfig); err != nil {
log.Println(err)
log.Fatalln("error loading pgp key, exiting")
}
log.Println("Indexing packages...")
for _, dist := range parsedConfig.DistroNames {
distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)}
for _, arch := range parsedConfig.SupportArch {
files, err := buildPackageList(parsedConfig, dist, arch)
if err != nil {
log.Fatalln("Unable to load packages:", err)
}
distro.Architectures[arch] = files
log.Println("Generating packages file for", dist, arch)
createPackagesCached(parsedConfig, dist, arch, files)
}
distros[dist] = distro
}
http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(parsedConfig.RootRepoPath))))
http.Handle("/upload", uploadHandler(parsedConfig))
http.Handle("/delete", deleteHandler(parsedConfig))
if parsedConfig.EnableSSL {
log.Println("running with SSL enabled")
log.Fatalln(http.ListenAndServeTLS(":"+parsedConfig.ListenPort, parsedConfig.SSLCert, parsedConfig.SSLKey, nil))
} else {
log.Println("running without SSL enabled")
log.Fatalln(http.ListenAndServe(":"+parsedConfig.ListenPort, nil))
}
}
func setupPgp(config Conf) error {
if config.PGPSecretKey == "" {
return nil
}
secretKey, err := os.Open(config.PGPSecretKey)
if err != nil {
return fmt.Errorf("failed to open private key ring file: %s", err)
}
defer secretKey.Close()
entitylist, err := openpgp.ReadKeyRing(secretKey)
if err != nil {
return fmt.Errorf("failed to read key ring: %s", err)
}
if len(entitylist) < 1 {
return errors.New("no keys in key ring")
}
pgpEntity = entitylist[0]
passphrase := []byte(config.PGPPassphrase)
if err := pgpEntity.PrivateKey.Decrypt(passphrase); err != nil {
return err
}
for _, subkey := range pgpEntity.Subkeys {
err := subkey.PrivateKey.Decrypt(passphrase)
if err != nil {
return err
}
}
return nil
}
func createDirs(config Conf) error {
for _, distro := range config.DistroNames {
for _, arch := range config.SupportArch {
if _, err := os.Stat(config.ArchPath(distro, arch)); err != nil {
if os.IsNotExist(err) {
log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)
if err := os.MkdirAll(config.ArchPath(distro, arch), 0755); err != nil {
return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err)
}
} else {
return fmt.Errorf("error inspecting %s (%s): %s", distro, arch, err)
}
}
if _, err := os.Stat(config.PoolPath(distro, arch)); err != nil {
if os.IsNotExist(err) {
log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)
if err := os.MkdirAll(config.PoolPath(distro, arch), 0755); err != nil {
return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err)
}
} else {
return fmt.Errorf("error inspecting %s (%s): %s", distro, arch, err)
}
}
}
}
return nil
}