tyler
/
deb-simple
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Cleanup, fixes, cloud fs support

This commit is contained in:
Tyler 2020-06-14 04:29:16 -04:00
parent 8bcb538e10
commit b651ccd250
11 changed files with 228 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.drone.yml
View File

@ -1,20 +1,13 @@
workspace:
base: /go
path: src/gitea.meow.tf/tyler/deb-simple
kind: pipeline
name: default
pipeline:
dependencies:
image: golang:latest
commands:
- mkdir /go/bin
- curl https://glide.sh/get | sh
- glide install
steps:
build-i386:
image: golang:latest
group: build
commands:
- mkdir -p build/i386
- GOOS=linux GOARCH=386 go build -o build/i386/deb-simple
- GOOS=linux GOARCH=386 go build -o /build/i386/deb-simple
build-amd64:
image: golang:latest
group: build
@ -26,13 +19,13 @@ pipeline:
group: build
commands:
- mkdir -p build/armv7
- GOOS=linux GOARCH=arm GOARM=7 go build -o build/armv7/deb-simple
- GOOS=linux GOARCH=arm GOARM=7 go build -o /build/armv7/deb-simple
build-arm64:
image: golang:latest
group: build
commands:
- mkdir -p build/arm64
- GOOS=linux GOARCH=arm64 go build -o build/arm64/deb-simple
- GOOS=linux GOARCH=arm64 go build -o /build/arm64/deb-simple
package:
image: tystuyfzand/fpm
commands:
@ -42,5 +35,8 @@ pipeline:
- ARCH=amd64 packaging/build-package.sh
- ARCH=armv7 packaging/build-package.sh
- ARCH=arm64 packaging/build-package.sh
- packaging/package-upload.sh
secrets: [ upload_url ]
secrets: [ upload_url ]
volumes:
- name: build
temp: {}

19
apt.go
View File

@ -6,11 +6,10 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
"gitea.meow.tf/tyler/deb-simple/deb/release"
"github.com/spf13/afero"
"golang.org/x/crypto/openpgp"
"io"
"io/ioutil"
"os"
"meow.tf/deb-simple/deb/release"
"path"
"strings"
"time"
@ -18,7 +17,7 @@ import (
func createRelease(config Conf, distro string) error {
outfile, err := os.Create(path.Join(config.DistPath(distro), "Release"))
outfile, err := fs.Create(path.Join(config.DistPath(distro), "Release"))
if err != nil {
return fmt.Errorf("failed to create Release: %s", err)
@ -36,7 +35,7 @@ func createRelease(config Conf, distro string) error {
for _, arch := range config.Repo.ArchitectureNames() {
absolutePath := path.Join(config.DistPath(distro), "main", "binary-" + arch)
list, err := ioutil.ReadDir(absolutePath)
list, err := afero.ReadDir(fs, absolutePath)
if err != nil {
continue
@ -51,16 +50,16 @@ func createRelease(config Conf, distro string) error {
fileLocalPath = strings.Replace(fileLocalPath, "\\", "/", -1)
f, err := os.Open(filePath)
f, err := fs.Open(filePath)
if err != nil {
return err
}
var size int64 = file.Size()
var size = file.Size()
if size == 0 {
if stat, err := os.Stat(filePath); err == nil {
if stat, err := fs.Stat(filePath); err == nil {
size = stat.Size()
}
}
@ -102,7 +101,7 @@ func createRelease(config Conf, distro string) error {
func signRelease(config Conf, distro string) error {
distPath := config.DistPath(distro)
f, err := os.Open(path.Join(distPath, "Release"))
f, err := fs.Open(path.Join(distPath, "Release"))
if err != nil {
return fmt.Errorf("failed to read Release: %s", err)
@ -110,7 +109,7 @@ func signRelease(config Conf, distro string) error {
defer f.Close()
gpgfile, err := os.Create(path.Join(distPath, "Release.gpg"))
gpgfile, err := fs.Create(path.Join(distPath, "Release.gpg"))
if err != nil {
return fmt.Errorf("failed to create Release.gpg: %s", err)

8
config.go
View File

@ -32,19 +32,19 @@ type PGPConf struct {
}
func (c Conf) DistPath(distro string) string {
return path.Join(c.Repo.Root, "dists", distro)
return path.Join("dists", distro)
}
func (c Conf) ArchPath(distro, arch string) string {
return path.Join(c.Repo.Root, "dists", distro, "main/binary-"+arch)
return path.Join("dists", distro, "main/binary-"+arch)
}
func (c Conf) PoolPath(distro, arch string) string {
return path.Join(c.Repo.Root, "pool/main", distro, arch)
return path.Join("pool/main", distro, arch)
}
func (c Conf) PoolPackagePath(distro, arch, name string) string {
return path.Join(c.Repo.Root, c.RelativePoolPackagePath(distro, arch, name))
return c.RelativePoolPackagePath(distro, arch, name)
}
func (c Conf) RelativePoolPackagePath(distro, arch, name string) string {

4
deb/release/release.go
View File

@ -1,9 +1,9 @@
package release
import (
"time"
"io"
"bufio"
"io"
"time"
)
type Release struct {

2
deb/release/release_test.go
View File

@ -1,8 +1,8 @@
package release
import (
"testing"
"bytes"
"testing"
"time"
)

4
deb/release/writer.go
View File

@ -1,9 +1,9 @@
package release
import (
"strings"
"fmt"
"bufio"
"fmt"
"strings"
)
type writer struct {

12
go.mod Normal file
View File

@ -0,0 +1,12 @@
module meow.tf/deb-simple
go 1.14
require (
github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2
github.com/blang/semver v3.5.1+incompatible
github.com/go-ini/ini v1.28.2
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/spf13/afero v1.2.2
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
)

261
http.go
View File

@ -12,6 +12,35 @@ import (
"strings"
)
func requireAuth(fn http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
key := extractAuthorization(r)
if key == "" || key != conf.Http.Key {
http.Error(w, "unauthorized", http.StatusForbidden)
return
}
fn(w, r)
}
}
func extractAuthorization(r *http.Request) string {
auth := r.Header.Get("Authorization")
if auth != "" {
idx := strings.Index(auth, " ")
if idx == -1 || auth[0:idx] != "Token" {
return ""
}
return auth[idx+1:]
}
return r.URL.Query().Get("key")
}
func rescanHandler(w http.ResponseWriter, r *http.Request) {
mutex.Lock()
defer mutex.Unlock()
@ -23,7 +52,7 @@ func rescanHandler(w http.ResponseWriter, r *http.Request) {
}
if _, exists := distros[distroName]; !exists {
httpErrorf(w, "Unable to find distro %s", distroName)
httpErrorf(w, http.StatusBadRequest, "invalid distro %s", distroName)
return
}
@ -57,13 +86,6 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
distroName = "stable"
}
key := r.URL.Query().Get("key")
if key == "" || key != conf.Http.Key {
http.Error(w, "unauthorized", 403)
return
}
force := false
forceStr := r.URL.Query().Get("force")
@ -75,7 +97,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
reader, err := r.MultipartReader()
if err != nil {
httpErrorf(w, "error creating multipart reader: %s", err)
httpErrorf(w, http.StatusInternalServerError, "error creating multipart reader: %s", err)
return
}
@ -84,7 +106,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
distro, exists := distros[distroName]
if !exists {
httpErrorf(w, "invalid distro: %s", distroName)
httpErrorf(w, http.StatusBadRequest, "invalid distro: %s", distroName)
mutex.RUnlock()
return
}
@ -97,21 +119,10 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
modifiedArches := make(map[string]bool)
baseDir := path.Join(os.TempDir(), "deb-simple")
if _, err := os.Stat(baseDir); err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(baseDir, 0755); err != nil {
httpErrorf(w, "error creating path: %s", err)
return
}
}
}
for {
part, err := reader.NextPart()
if err == io.EOF {
if err != nil {
break
}
@ -119,96 +130,14 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
continue
}
tempFile := path.Join(baseDir, part.FileName())
dst, err := os.Create(tempFile)
archType, err := loadAndCheckPackage(distroName, distro, part.FileName(), part, force)
if err != nil {
httpErrorf(w, "error creating deb file: %s", err)
httpErrorf(w, http.StatusInternalServerError, err.Error())
return
}
if _, err := io.Copy(dst, part); err != nil {
dst.Close()
httpErrorf(w, "error writing deb file: %s", err)
return
}
dst.Close()
// Get package name, if it already exists remove the old file.
f, err := newPackageFile(tempFile)
if err != nil {
httpErrorf(w, "error loading package info: %s", err)
}
archType := f.Info.Architecture
if archType == "" {
archType = queryArchType
}
modifiedArches[archType] = true
// Get current packages
packages, exists := distro.Architectures[archType]
if !exists {
httpErrorf(w, "invalid arch: %s", archType)
continue
}
// New path based off package data
newPath := conf.PoolPackagePath(distroName, archType, f.Info.Package)
if _, err := os.Stat(newPath); err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(newPath, 0755); err != nil {
httpErrorf(w, "error creating path: %s", err)
continue
}
}
}
f.Path = path.Join(conf.RelativePoolPackagePath(distroName, archType, f.Info.Package), part.FileName())
if err := copyFile(tempFile, path.Join(newPath, part.FileName())); err != nil {
httpErrorf(w, "error copying temporary file: %s", err)
continue
}
if err := os.Remove(tempFile); err != nil {
httpErrorf(w, "unable to remove temporary file: %s", err)
continue
}
if p, exists := packages[f.Info.Package]; exists {
v1, err1 := semver.Parse(p.Info.Version)
v2, err2 := semver.Parse(f.Info.Version)
if err1 == nil && err2 == nil && v1.Compare(v2) > 0 && !force {
// Don't replace newer package
httpErrorf(w, "version in old package is greater than new: %s, %s - override with \"force\"", p.Info.Version, f.Info.Version)
continue
}
// Archive old file
log.Println("Replacing", p.Name, "with", f.Name)
oldPath := path.Join(conf.Repo.Root, p.Path)
// If oldPath == newPath then we already overwrote it
if oldPath != newPath {
if err := os.Remove(oldPath); err != nil && !os.IsNotExist(err) {
httpErrorf(w, "Unable to remove old package: %s", err)
continue
}
}
}
packages[f.Info.Package] = f
}
log.Println("got lock, updating package list...")
@ -217,7 +146,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
for archType, _ := range modifiedArches {
if err := createPackagesCached(conf, distroName, archType, distro.Architectures[archType]); err != nil {
httpErrorf(w, "error creating package: %s", err)
httpErrorf(w, http.StatusInternalServerError, "error creating package: %s", err)
return
}
}
@ -225,20 +154,109 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
err = createRelease(conf, distroName)
if err != nil {
httpErrorf(w, "error creating package: %s", err)
httpErrorf(w, http.StatusInternalServerError, "error creating package: %s", err)
return
}
err = saveCache(distro)
if err != nil {
httpErrorf(w, "error updating cache: %s", err)
httpErrorf(w, http.StatusInternalServerError, "error updating cache: %s", err)
return
}
w.WriteHeader(http.StatusCreated)
}
func copyToTemp(fileName string, r io.Reader) (string, error) {
tempFile := path.Join(baseTempDir, fileName)
dst, err := os.Create(tempFile)
if err != nil {
return "", fmt.Errorf("error creating deb file: %s", err)
}
defer dst.Close()
if _, err := io.Copy(dst, r); err != nil {
return "", fmt.Errorf("error creating deb file: %s", err)
}
return fileName, nil
}
func loadAndCheckPackage(distroName string, distro *Distro, fileName string, r io.Reader, force bool) (string, error) {
tempFile, err := copyToTemp(fileName, r)
if err != nil {
return "", err
}
// Get package name, if it already exists remove the old file.
f, err := newPackageFile(tempFile)
if err != nil {
return "", fmt.Errorf("error loading package info: %s", err)
}
archType := f.Info.Architecture
// Get current packages
packages, exists := distro.Architectures[archType]
if !exists {
return "", fmt.Errorf("invalid arch: %s", archType)
}
// New path based off package data
newPath := conf.PoolPackagePath(distroName, archType, f.Info.Package)
if _, err := fs.Stat(newPath); err != nil {
if os.IsNotExist(err) {
if err := fs.MkdirAll(newPath, 0755); err != nil {
return "", fmt.Errorf("error creating path: %s", err)
}
}
}
f.Path = path.Join(conf.RelativePoolPackagePath(distroName, archType, f.Info.Package), fileName)
if err := copyFile(tempFile, path.Join(newPath, fileName)); err != nil {
return "", fmt.Errorf("error copying temporary file: %s", err)
}
if err := fs.Remove(tempFile); err != nil {
return "", fmt.Errorf("unable to remove temporary file: %s", err)
}
if p, exists := packages[f.Info.Package]; exists {
v1, err1 := semver.Parse(p.Info.Version)
v2, err2 := semver.Parse(f.Info.Version)
if err1 == nil && err2 == nil {
if v1.Compare(v2) > 0 && !force {
// Don't replace newer package
return "", fmt.Errorf("version in old package is greater than new: %s, %s - override with \"force\"", p.Info.Version, f.Info.Version)
}
}
// Archive old file
log.Println("Replacing", p.Name, "with", f.Name)
// If oldPath == newPath then we already overwrote it
if path.Base(p.Path) != path.Base(newPath) {
if err := fs.Remove(p.Path); err != nil && !os.IsNotExist(err) {
return "", fmt.Errorf("unable to remove old package: %s", err)
}
}
}
packages[f.Info.Package] = f
return archType, nil
}
func deleteHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != "DELETE" {
http.Error(w, "method not supported", http.StatusMethodNotAllowed)
@ -247,7 +265,7 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) {
var req DeleteObj
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
httpErrorf(w, "failed to decode json: %s", err)
httpErrorf(w, http.StatusBadRequest, "failed to decode json: %s", err)
return
}
@ -256,7 +274,7 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) {
distro, exists := distros[req.DistroName]
if !exists {
httpErrorf(w, "invalid distro: %s", req.DistroName)
httpErrorf(w, http.StatusBadRequest, "invalid distro: %s", req.DistroName)
mutex.RUnlock()
return
}
@ -264,24 +282,17 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) {
packages, exists := distro.Architectures[req.Arch]
if !exists {
httpErrorf(w, "invalid arch: %s", req.Arch)
httpErrorf(w, http.StatusBadRequest, "invalid arch: %s", req.Arch)
mutex.RUnlock()
return
}
mutex.RUnlock()
key := r.URL.Query().Get("key")
if key == "" || key != conf.Http.Key {
http.Error(w, "unauthorized", http.StatusForbidden)
return
}
debPath := path.Join(conf.ArchPath(req.DistroName, req.Arch), req.Filename)
if err := os.Remove(debPath); err != nil {
httpErrorf(w, "failed to delete: %s", err)
if err := fs.Remove(debPath); err != nil {
httpErrorf(w, http.StatusInternalServerError, "failed to delete: %s", err)
return
}
@ -289,20 +300,20 @@ func deleteHandler(w http.ResponseWriter, r *http.Request) {
defer mutex.Unlock()
if err := createPackagesCached(conf, req.DistroName, req.Arch, packages); err != nil {
httpErrorf(w, "failed to delete package: %s", err)
httpErrorf(w, http.StatusInternalServerError, "failed to delete package: %s", err)
return
}
if err := createRelease(conf, req.DistroName); err != nil {
httpErrorf(w, "failed to delete package: %s", err)
httpErrorf(w, http.StatusInternalServerError, "failed to delete package: %s", err)
return
}
w.WriteHeader(http.StatusOK)
}
func httpErrorf(w http.ResponseWriter, format string, a ...interface{}) {
func httpErrorf(w http.ResponseWriter, code int, format string, a ...interface{}) {
err := fmt.Errorf(format, a...)
log.Println(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
http.Error(w, err.Error(), code)
}

44
packages.go
View File

@ -3,18 +3,16 @@ package main
import (
"bufio"
"bytes"
"compress/gzip"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"gitea.meow.tf/tyler/deb-simple/deb/archive"
"github.com/blang/semver"
"github.com/spf13/afero"
"io"
"io/ioutil"
"os"
"meow.tf/deb-simple/deb/archive"
"path"
"regexp"
"strings"
@ -68,7 +66,7 @@ var (
)
func loadCache(dist string) error {
f, err := os.Open(path.Join(conf.DistPath(dist), "dist.json"))
f, err := fs.Open(path.Join(conf.DistPath(dist), "dist.json"))
if err != nil {
return err
@ -88,7 +86,7 @@ func loadCache(dist string) error {
}
func saveCache(dist *Distro) error {
f, err := os.Create(path.Join(conf.DistPath(dist.Name), "dist.json"))
f, err := fs.Create(path.Join(conf.DistPath(dist.Name), "dist.json"))
if err != nil {
return err
@ -114,7 +112,7 @@ func buildPackageList(config Conf, distro, arch string) (map[string]*PackageFile
}
func scanRecursive(base string, m map[string]*PackageFile) error {
dirList, err := ioutil.ReadDir(base)
dirList, err := afero.ReadDir(fs, base)
if err != nil {
return err
@ -170,11 +168,11 @@ func newPackageFile(filePath string) (*PackageFile, error) {
p.Info = parsePackageData(p.ControlData)
if stat, err := os.Stat(filePath); err == nil {
if stat, err := fs.Stat(filePath); err == nil {
p.Size = stat.Size()
}
f, err := os.Open(filePath)
f, err := fs.Open(filePath)
if err != nil {
return nil, err
@ -202,21 +200,14 @@ func newPackageFile(filePath string) (*PackageFile, error) {
}
func createPackagesCached(config Conf, distro, arch string, packages map[string]*PackageFile) error {
stdFile, err := os.Create(path.Join(config.ArchPath(distro, arch), "Packages"))
stdFile, err := fs.Create(path.Join(config.ArchPath(distro, arch), "Packages"))
if err != nil {
return fmt.Errorf("failed to create packages: %s", err)
}
defer stdFile.Close()
gzipFile, err := os.Create(path.Join(config.ArchPath(distro, arch), "Packages.gz"))
if err != nil {
return fmt.Errorf("failed to create packages.gz: %s", err)
}
defer gzipFile.Close()
gzWriter := gzip.NewWriter(gzipFile)
defer gzWriter.Close()
stdOut := bufio.NewWriter(stdFile)
// loop through each directory
@ -238,12 +229,23 @@ func createPackagesCached(config Conf, distro, arch string, packages map[string]
packBuf.WriteString("\n\n")
stdOut.Write(packBuf.Bytes())
gzWriter.Write(packBuf.Bytes())
}
stdOut.Flush()
gzWriter.Flush()
gzipFile, err := fs.Create(path.Join(config.ArchPath(distro, arch), "Packages.gz"))
if err != nil {
return fmt.Errorf("failed to create packages.gz: %s", err)
}
defer gzipFile.Close()
stdFile.Seek(0, io.SeekStart)
if _, err = io.Copy(gzipFile, stdFile); err != nil {
return err
}
return nil
}

2
packaging/build-package.sh
View File

@ -1,4 +1,4 @@
fpm -s dir -t deb -p build/$ARCH/deb-simple_$VERSION.deb \
fpm -s dir -t deb -p /build/$ARCH/deb-simple_$VERSION_$ARCH.deb \
-n deb-simple -v $VERSION \
--config-files /etc/deb-simple.conf \
--deb-priority optional --force \

36
server.go
View File

@ -5,12 +5,14 @@ import (
"flag"
"fmt"
"github.com/go-ini/ini"
"github.com/spf13/afero"
"golang.org/x/crypto/openpgp"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path"
"runtime"
"strings"
"sync"
@ -37,6 +39,8 @@ var (
flagShowVersion = flag.Bool("version", false, "Show deb-simple version")
conf = Conf{}
pgpEntity *openpgp.Entity
fs afero.Fs
baseTempDir string
)
func main() {
@ -57,6 +61,18 @@ func main() {
log.Fatalln("unable to marshal config file, exiting...", err)
}
baseTempDir = path.Join(os.TempDir(), "deb-simple")
if _, err := os.Stat(baseTempDir); err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(baseTempDir, 0755); err != nil {
return
}
}
}
fs = afero.NewBasePathFs(afero.NewOsFs(), conf.Repo.Root)
if err := createDirs(conf); err != nil {
log.Println(err)
log.Fatalln("error creating directory structure, exiting")
@ -82,10 +98,12 @@ func main() {
mux := http.NewServeMux()
mux.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(conf.Repo.Root))))
mux.HandleFunc("/rescan", rescanHandler)
mux.HandleFunc("/upload", uploadHandler)
mux.HandleFunc("/delete", deleteHandler)
httpFs := afero.NewHttpFs(fs)
mux.Handle("/", http.StripPrefix("/", http.FileServer(httpFs)))
mux.HandleFunc("/rescan", requireAuth(rescanHandler))
mux.HandleFunc("/upload", requireAuth(uploadHandler))
mux.HandleFunc("/delete", requireAuth(deleteHandler))
bind := fmt.Sprintf(":%d", conf.Http.Port)
@ -161,20 +179,20 @@ func setupPgp(config Conf) error {
func createDirs(config Conf) error {
for _, distro := range config.Repo.DistroNames() {
for _, arch := range config.Repo.ArchitectureNames() {
if _, err := os.Stat(config.ArchPath(distro, arch)); err != nil {
if _, err := fs.Stat(config.ArchPath(distro, arch)); err != nil {
if os.IsNotExist(err) {
log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)
if err := os.MkdirAll(config.ArchPath(distro, arch), 0755); err != nil {
if err := fs.MkdirAll(config.ArchPath(distro, arch), 0755); err != nil {
return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err)
}
} else {
return fmt.Errorf("error inspecting %s (%s): %s", distro, arch, err)
}
}
if _, err := os.Stat(config.PoolPath(distro, arch)); err != nil {
if _, err := fs.Stat(config.PoolPath(distro, arch)); err != nil {
if os.IsNotExist(err) {
log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)
if err := os.MkdirAll(config.PoolPath(distro, arch), 0755); err != nil {
if err := fs.MkdirAll(config.PoolPath(distro, arch), 0755); err != nil {
return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err)
}
} else {
@ -202,7 +220,7 @@ func copyFile(oldPath, newPath string) error {
defer old.Close()
n, err := os.Create(newPath)
n, err := fs.Create(newPath)
if err != nil {
return err