diff --git a/.drone.yml b/.drone.yml index f18b45c..c18b428 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2,39 +2,15 @@ kind: pipeline name: default steps: - - name: build-i386 + - name: build image: golang:latest group: build commands: - mkdir -p /build/i386 - GOOS=linux GOARCH=386 go build -o /build/i386/deb-simple - volumes: - - name: build - path: /build - - name: build-amd64 - image: golang:latest - group: build - commands: - - mkdir -p /build/amd64 - - go build -o /build/amd64/deb-simple\ - volumes: - - name: build - path: /build - - name: build-armv7 - image: golang:latest - group: build - commands: - - mkdir -p /build/armv7 - - GOOS=linux GOARCH=arm GOARM=7 go build -o /build/armv7/deb-simple - volumes: - - name: build - path: /build - - name: build-arm64 - image: golang:latest - group: build - commands: - - mkdir -p /build/arm64 - - GOOS=linux GOARCH=arm64 go build -o /build/arm64/deb-simple + - GOOS=linux GOARCH=amd64 go build -o /build/amd64/deb-simple + - GOOS=linux GOARCH=arm GOARM=7 go build -o /build/armv7/deb-simple + - GOOS=linux GOARCH=arm64 go build -o /build/arm64/deb-simple volumes: - name: build path: /build diff --git a/config.go b/config.go index cdb763d..789d17e 100644 --- a/config.go +++ b/config.go @@ -14,6 +14,7 @@ type Conf struct { type FsConf struct { Driver string `ini:"driver"` + TmpDriver string `ini:"tmpDriver"` S3 struct { Bucket string `ini:"bucket"` Region string `ini:"region"` diff --git a/deb/archive/archive.go b/deb/archive/archive.go index 4ce2990..1c49dfa 100644 --- a/deb/archive/archive.go +++ b/deb/archive/archive.go @@ -8,19 +8,11 @@ import ( "github.com/blakesmith/ar" "io" "log" - "os" "strings" ) -func InspectPackage(filename string) (string, error) { - f, err := os.Open(filename) - - if err != nil { - return "", fmt.Errorf("error opening package file %s: %s", filename, err) - } - - arReader := ar.NewReader(f) - defer f.Close() +func InspectPackage(r io.Reader) (string, error) { + arReader := ar.NewReader(r) for { header, err := arReader.Next() @@ -37,6 +29,7 @@ func InspectPackage(filename string) (string, error) { return InspectPackageControl(arReader) } } + return "", nil } diff --git a/glide.lock b/glide.lock deleted file mode 100644 index ebb096e..0000000 --- a/glide.lock +++ /dev/null @@ -1,20 +0,0 @@ -hash: 1a6d5cdf757d6be4cab4e8b0d2224a38038fb086e135b15282ba85d02c54368a -updated: 2017-09-14T00:26:28.2385236-04:00 -imports: -- name: github.com/blakesmith/ar - version: 8bd4349a67f2533b078dbc524689d15dba0f4659 -- name: github.com/blang/semver - version: 2ee87856327ba09384cabd113bc6b5d174e9ec0f -- name: github.com/go-ini/ini - version: 20b96f641a5ea98f2f8619ff4f3e061cff4833bd -- name: golang.org/x/crypto - version: 7e9105388ebff089b3f99f0ef676ea55a6da3a7e - subpackages: - - cast5 - - openpgp - - openpgp/armor - - openpgp/elgamal - - openpgp/errors - - openpgp/packet - - openpgp/s2k -testImports: [] diff --git a/glide.yaml b/glide.yaml deleted file mode 100644 index 0836609..0000000 --- a/glide.yaml +++ /dev/null @@ -1,10 +0,0 @@ -package: meow.tf/deb-simple -import: -- package: github.com/blakesmith/ar -- package: golang.org/x/crypto - subpackages: - - openpgp -- package: github.com/blang/semver - version: ^3.5.0 -- package: github.com/go-ini/ini - version: ^1.28.0 diff --git a/go.mod b/go.mod index 80a05d5..036513e 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,12 @@ module meow.tf/deb-simple go 1.14 require ( + github.com/aws/aws-sdk-go v1.31.3 github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 github.com/blang/semver v3.5.1+incompatible - github.com/fclairamb/afero-s3 v0.1.0 // indirect + github.com/fclairamb/afero-s3 v0.1.0 github.com/go-ini/ini v1.28.2 + github.com/sirupsen/logrus v1.6.0 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/afero v1.2.2 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 diff --git a/go.sum b/go.sum index 0ce8951..27f4c33 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,7 @@ github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2/go.mod h1:PkYb9DJNAw github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fclairamb/afero-s3 v0.1.0 h1:WfPD2Px6AlcEBw1jUmJC295wpTXkfglYtzbSdTfaNzg= github.com/fclairamb/afero-s3 v0.1.0/go.mod h1:HHcYBsOLXjeqdnKQ61Rx43+vj+jGe9DjYTQbKhendwQ= github.com/go-ini/ini v1.28.2 h1:drmmYv7psRpoGZkPtPKKTB+ZFSnvmwCMfNj5o1nLh2Y= @@ -16,8 +17,12 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -25,6 +30,7 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= golang.org/x/crypto v0.0.0-20170523101029-7e9105388ebf h1:Ak4rvIfvuVxAE1NgtCq1GvH0e5psMKmzowfoGdL8QBA= golang.org/x/crypto v0.0.0-20170523101029-7e9105388ebf/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -33,6 +39,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= diff --git a/http.go b/http.go index a3732a1..6e56a25 100644 --- a/http.go +++ b/http.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" "github.com/blang/semver" + log "github.com/sirupsen/logrus" "io" - "log" "net/http" "os" "path" @@ -140,17 +140,23 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { modifiedArches[archType] = true } - log.Println("got lock, updating package list...") - // Recreate the package index and release file. for archType, _ := range modifiedArches { + log.WithFields(log.Fields{ + "arch": archType, + }).Debug("Updating package list...") + if err := createPackagesCached(conf, distroName, archType, distro.Architectures[archType]); err != nil { httpErrorf(w, http.StatusInternalServerError, "error creating package: %s", err) return } } + log.WithFields(log.Fields{ + "distroName": distroName, + }).Debug("Updating release files...") + err = createRelease(conf, distroName) if err != nil { @@ -158,6 +164,8 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { return } + log.Debug("Dumping cache...") + err = saveCache(distro) if err != nil { @@ -165,13 +173,13 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { return } + log.Debug("Done.") + w.WriteHeader(http.StatusCreated) } func copyToTemp(fileName string, r io.Reader) (string, error) { - tempFile := path.Join(baseTempDir, fileName) - - dst, err := os.Create(tempFile) + dst, err := tmpFs.Create(fileName) if err != nil { return "", fmt.Errorf("error creating deb file: %s", err) @@ -194,7 +202,7 @@ func loadAndCheckPackage(distroName string, distro *Distro, fileName string, r i } // Get package name, if it already exists remove the old file. - f, err := newPackageFile(tempFile) + f, err := newPackageFile(tmpFs, tempFile) if err != nil { return "", fmt.Errorf("error loading package info: %s", err) @@ -222,11 +230,11 @@ func loadAndCheckPackage(distroName string, distro *Distro, fileName string, r i f.Path = path.Join(conf.RelativePoolPackagePath(distroName, archType, f.Info.Package), fileName) - if err := copyFile(tempFile, path.Join(newPath, fileName)); err != nil { + if err := copyFile(tmpFs, tempFile, fs, path.Join(newPath, fileName)); err != nil { return "", fmt.Errorf("error copying temporary file: %s", err) } - if err := fs.Remove(tempFile); err != nil { + if err := tmpFs.Remove(tempFile); err != nil { return "", fmt.Errorf("unable to remove temporary file: %s", err) } @@ -242,10 +250,13 @@ func loadAndCheckPackage(distroName string, distro *Distro, fileName string, r i } // Archive old file - log.Println("Replacing", p.Name, "with", f.Name) + log.WithFields(log.Fields{ + "existing": p.Path, + "new": f.Path, + }).Debug("Replacing existing file") // If oldPath == newPath then we already overwrote it - if path.Base(p.Path) != path.Base(newPath) { + if path.Base(p.Path) != path.Base(f.Path) { if err := fs.Remove(p.Path); err != nil && !os.IsNotExist(err) { return "", fmt.Errorf("unable to remove old package: %s", err) } diff --git a/packages.go b/packages.go index 6714f84..c0763a5 100644 --- a/packages.go +++ b/packages.go @@ -130,7 +130,7 @@ func scanRecursive(base string, m map[string]*PackageFile) error { continue } - p, err := newPackageFile(path.Join(base, file.Name())) + p, err := newPackageFile(fs, path.Join(base, file.Name())) if err != nil { return err @@ -152,27 +152,23 @@ func scanRecursive(base string, m map[string]*PackageFile) error { return nil } -func newPackageFile(filePath string) (*PackageFile, error) { +func newPackageFile(sourceFs afero.Fs, filePath string) (*PackageFile, error) { p := &PackageFile{ Name: path.Base(filePath), - Path: stripPrefix(filePath, conf.Repo.Root), + Path: filePath, } var err error - p.ControlData, err = archive.InspectPackage(filePath) + stat, err := sourceFs.Stat(filePath) if err != nil { return nil, err } - p.Info = parsePackageData(p.ControlData) + p.Size = stat.Size() - if stat, err := fs.Stat(filePath); err == nil { - p.Size = stat.Size() - } - - f, err := fs.Open(filePath) + f, err := sourceFs.Open(filePath) if err != nil { return nil, err @@ -180,6 +176,18 @@ func newPackageFile(filePath string) (*PackageFile, error) { defer f.Close() + p.ControlData, err = archive.InspectPackage(f) + + p.Info = parsePackageData(p.ControlData) + + if err != nil { + return nil, err + } + + if _, err = f.Seek(0, io.SeekStart); err != nil { + return nil, err + } + var ( md5hash = md5.New() sha1hash = sha1.New() diff --git a/server.go b/server.go index f53aa39..c2c8c8b 100644 --- a/server.go +++ b/server.go @@ -9,11 +9,11 @@ import ( "github.com/aws/aws-sdk-go/aws/session" s3 "github.com/fclairamb/afero-s3" "github.com/go-ini/ini" + log "github.com/sirupsen/logrus" "github.com/spf13/afero" "golang.org/x/crypto/openpgp" "io" "io/ioutil" - "log" "net/http" "os" "path" @@ -41,15 +41,20 @@ var ( mutex sync.RWMutex configFile = flag.String("c", "conf.ini", "config file location") flagShowVersion = flag.Bool("version", false, "Show deb-simple version") + flagDebug = flag.Bool("debug", false, "debug output") conf = Conf{} pgpEntity *openpgp.Entity fs afero.Fs - baseTempDir string + tmpFs afero.Fs ) func main() { flag.Parse() + if *flagDebug { + log.SetLevel(log.DebugLevel) + } + if *flagShowVersion { fmt.Printf("deb-simple %s (%s)\n", VERSION, runtime.Version()) os.Exit(0) @@ -58,23 +63,14 @@ func main() { file, err := ioutil.ReadFile(*configFile) if err != nil { - log.Fatalln("unable to read config file, exiting...") + log.WithError(err).Fatalln("unable to read config file, exiting...") } if err := ini.MapTo(&conf, file); err != nil { - log.Fatalln("unable to marshal config file, exiting...", err) - } - - baseTempDir = path.Join(os.TempDir(), "deb-simple") - - if _, err := os.Stat(baseTempDir); err != nil { - if os.IsNotExist(err) { - if err := os.MkdirAll(baseTempDir, 0755); err != nil { - return - } - } + log.WithError(err).Fatalln("unable to marshal config file, exiting...", err) } + osFs := afero.NewOsFs() switch conf.Fs.Driver { case "s3": @@ -97,24 +93,41 @@ func main() { case "local": fallthrough default: - fs = afero.NewBasePathFs(afero.NewOsFs(), conf.Repo.Root) + fs = afero.NewBasePathFs(osFs, conf.Repo.Root) + } + + switch conf.Fs.TmpDriver { + case "memory": + tmpFs = afero.NewMemMapFs() + case "local": + fallthrough + default: + baseTempDir := path.Join(os.TempDir(), "deb-simple") + + if _, err := os.Stat(baseTempDir); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(baseTempDir, 0755); err != nil { + log.Fatalln("Unable to create temp dir:", err) + } + } + } + + tmpFs = afero.NewBasePathFs(osFs, baseTempDir) } if err := createDirs(conf); err != nil { - log.Println(err) - log.Fatalln("error creating directory structure, exiting") + log.WithError(err).Fatalln("error creating directory structure, exiting") } if err := setupPgp(conf); err != nil { - log.Println(err) - log.Fatalln("error loading pgp key, exiting") + log.WithError(err).Fatalln("error loading pgp key, exiting") } - log.Println("Indexing packages...") + log.Info("Indexing packages...") for _, dist := range conf.Repo.DistroNames() { if err := loadCache(dist); err != nil { - log.Println("Unable to load cached data for", dist, "- reindexing") + log.Info("Unable to load cached data for", dist, "- reindexing") distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)} go scanInitialPackages(conf, distro) @@ -135,10 +148,10 @@ func main() { bind := fmt.Sprintf(":%d", conf.Http.Port) if conf.Http.SSL { - log.Println("running with SSL enabled") + log.Info("running with SSL enabled") log.Fatalln(http.ListenAndServeTLS(bind, conf.Http.SSLCert, conf.Http.SSLKey, mux)) } else { - log.Println("running without SSL enabled") + log.Info("running without SSL enabled") log.Fatalln(http.ListenAndServe(bind, mux)) } } @@ -153,14 +166,27 @@ func scanInitialPackages(config Conf, dist *Distro) { dist.Architectures[arch] = files - log.Println("Generating packages file for", dist.Name, arch) - createPackagesCached(config, dist.Name, arch, files) + log.Info("Generating packages file for", dist.Name, arch) + + err = createPackagesCached(config, dist.Name, arch, files) + + if err != nil { + log.WithError(err).Fatalln("Unable to generate package list") + } } - log.Println("Generating Release for", dist.Name) - createRelease(config, dist.Name) + log.Info("Generating Release for", dist.Name) + err := createRelease(config, dist.Name) - saveCache(dist) + if err != nil { + log.WithError(err).Fatalln("Unable to generate release file") + } + + err = saveCache(dist) + + if err != nil { + log.WithError(err).Fatalln("Unable to save cache") + } } func setupPgp(config Conf) error { @@ -208,7 +234,7 @@ func createDirs(config Conf) error { for _, arch := range config.Repo.ArchitectureNames() { if _, err := fs.Stat(config.ArchPath(distro, arch)); err != nil { if os.IsNotExist(err) { - log.Printf("Directory for %s (%s) does not exist, creating", distro, arch) + log.Debugf("Directory for %s (%s) does not exist, creating", distro, arch) if err := fs.MkdirAll(config.ArchPath(distro, arch), 0755); err != nil { return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err) } @@ -218,7 +244,7 @@ func createDirs(config Conf) error { } if _, err := fs.Stat(config.PoolPath(distro, arch)); err != nil { if os.IsNotExist(err) { - log.Printf("Directory for %s (%s) does not exist, creating", distro, arch) + log.Debugf("Directory for %s (%s) does not exist, creating", distro, arch) if err := fs.MkdirAll(config.PoolPath(distro, arch), 0755); err != nil { return fmt.Errorf("error creating directory for %s (%s): %s", distro, arch, err) } @@ -231,15 +257,8 @@ func createDirs(config Conf) error { return nil } -func stripPrefix(str, prefix string) string { - if strings.Index(str, prefix) == 0 { - return strings.TrimLeft(str[len(prefix):], "/") - } - return str -} - -func copyFile(oldPath, newPath string) error { - old, err := os.Open(oldPath) +func copyFile(sourceFs afero.Fs, oldPath string, destFs afero.Fs, newPath string) error { + old, err := sourceFs.Open(oldPath) if err != nil { return err @@ -247,7 +266,7 @@ func copyFile(oldPath, newPath string) error { defer old.Close() - n, err := fs.Create(newPath) + n, err := destFs.Create(newPath) if err != nil { return err