diff --git a/.gitignore b/.gitignore index 504ea6d..1ef57b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /deb-simple +/simple /repo vendor -conf.json +conf.ini *.pprof diff --git a/sample_conf.ini b/sample_conf.ini index d547fce..7d80c97 100644 --- a/sample_conf.ini +++ b/sample_conf.ini @@ -1,12 +1,13 @@ -{ - "listenPort" : "9090", - "rootRepoPath" : "/opt/deb-simple/repo", - "supportedArch" : ["all","i386","amd64"], - "distroNames":["stable"], - "pgpSecretKey": "secring.gpg", - "pgpPassphrase" : "", - "enableSSL" : false, - "SSLcert" : "server.crt", - "SSLkey" : "server.key", - "key" : "abcdefg" -} +[http] +port = 9090 +key = "abcdefg" +ssl = false + +[repo] +root = "repo/" +distros = "stable" +architectures = "all,i386,amd64" + +[pgp] +key = "" +passphrase = "" \ No newline at end of file diff --git a/src/meow.tf/deb-simple/config.go b/src/meow.tf/deb-simple/config.go index 2c29f4c..639e93d 100644 --- a/src/meow.tf/deb-simple/config.go +++ b/src/meow.tf/deb-simple/config.go @@ -1 +1,56 @@ package simple + +import ( + "path/filepath" + "strings" +) + +type Conf struct { + Http HttpConf `ini:"http"` + Repo RepoConf `ini:"repo"` + PGP PGPConf `ini:"pgp"` +} + +type HttpConf struct { + Port int `ini:"port"` + Key string `ini:"key"` + SSL bool `ini:"ssl"` + SSLCert string `ini:"cert"` + SSLKey string `ini:"key"` +} + +type RepoConf struct { + Root string `ini:"root"` + Distros string `ini:"distros"` + Architectures string `ini:"architectures"` +} + +type PGPConf struct { + Key string `ini:"key"` + Passphrase string `ini:"passphrase"` +} + +func (c Conf) DistPath(distro string) string { + return filepath.Join(c.Repo.Root, "dists", distro) +} + +func (c Conf) ArchPath(distro, arch string) string { + return filepath.Join(c.Repo.Root, "dists", distro, "main/binary-"+arch) +} + +func (c Conf) PoolPath(distro, arch string) string { + return filepath.Join(c.Repo.Root, "pool/main", distro, arch) +} + +func (c Conf) PoolPackagePath(distro, arch, name string) string { + name = packageName(name) + return filepath.Join(c.Repo.Root, "pool/main", distro, arch, name[0:1], name) +} + +func (c RepoConf) DistroNames() []string { + return strings.Split(c.Distros, ",") +} + +func (c RepoConf) ArchitectureNames() []string { + return strings.Split(c.Architectures, ",") +} \ No newline at end of file diff --git a/src/meow.tf/deb-simple/glide.lock b/src/meow.tf/deb-simple/glide.lock index 2d46210..0d85bf8 100644 --- a/src/meow.tf/deb-simple/glide.lock +++ b/src/meow.tf/deb-simple/glide.lock @@ -1,10 +1,12 @@ -hash: 79d0f0e99e99ce9318bb0e8c6ab398657a20679dbe3c421c6fc588db7a738131 -updated: 2017-06-11T03:04:15.102617-04:00 +hash: 33760adb1090508f6d56dfc1d600a664790a280e5fc4d84cb26a7e11ad0f1f35 +updated: 2017-06-11T20:13:27.6668377-04:00 imports: - name: github.com/blakesmith/ar version: 8bd4349a67f2533b078dbc524689d15dba0f4659 - name: github.com/blang/semver version: b38d23b8782a487059e8fc8773e9a5b228a77cb6 +- name: github.com/go-ini/ini + version: d3de07a94d22b4a0972deb4b96d790c2c0ce8333 - name: golang.org/x/crypto version: 7e9105388ebff089b3f99f0ef676ea55a6da3a7e subpackages: diff --git a/src/meow.tf/deb-simple/glide.yaml b/src/meow.tf/deb-simple/glide.yaml index eaa6c8e..de8d478 100644 --- a/src/meow.tf/deb-simple/glide.yaml +++ b/src/meow.tf/deb-simple/glide.yaml @@ -6,3 +6,5 @@ import: - openpgp - package: github.com/blang/semver version: ^3.5.0 +- package: github.com/go-ini/ini + version: ^1.28.0 diff --git a/src/meow.tf/deb-simple/http.go b/src/meow.tf/deb-simple/http.go index d86f75f..ba82348 100644 --- a/src/meow.tf/deb-simple/http.go +++ b/src/meow.tf/deb-simple/http.go @@ -11,214 +11,232 @@ import ( "github.com/blang/semver" ) -func uploadHandler(config Conf) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { - http.Error(w, "method not supported", http.StatusMethodNotAllowed) - return - } - archType := r.URL.Query().Get("arch") +func rescanHandler(w http.ResponseWriter, r *http.Request) { + distroName := r.URL.Query().Get("distro") - if archType == "" { - archType = "all" - } + if distroName == "" { + distroName = "stable" + } - distroName := r.URL.Query().Get("distro") + if _, exists := distros[distroName]; !exists { + httpErrorf(w, "Unable to find distro %s", distroName) + return + } - if distroName == "" { - distroName = "stable" - } + distro := &Distro{Name: distroName, Architectures: make(map[string]map[string]*PackageFile)} - key := r.URL.Query().Get("key") + scanInitialPackages(conf, distro) - if key == "" || key != config.Key { - http.Error(w, "unauthorized", 403) - return - } - - force := false - - forceStr := r.URL.Query().Get("force") - - if forceStr != "" && forceStr == "true" { - force = true - } - - reader, err := r.MultipartReader() - - if err != nil { - httpErrorf(w, "error creating multipart reader: %s", err) - return - } - - mutex.RLock() - - distro, exists := distros[distroName] - - if !exists { - httpErrorf(w, "invalid distro: %s", distroName) - mutex.RUnlock() - return - } - - packages, exists := distro.Architectures[archType] - - if !exists { - httpErrorf(w, "invalid arch: %s", archType) - mutex.RUnlock() - return - } - - mutex.RUnlock() - - // Lock to prevent concurrent modification - mutex.Lock() - defer mutex.Unlock() - - for { - part, err := reader.NextPart() - - if err == io.EOF { - break - } - - if part.FileName() == "" { - continue - } - newPath := config.PoolPackagePath(distroName, archType, part.FileName()) - - if _, err := os.Stat(newPath); err != nil { - if os.IsNotExist(err) { - if err := os.MkdirAll(newPath, 0755); err != nil { - httpErrorf(w, "error creating path: %s", err) - return - } - } - } - - dst, err := os.Create(filepath.Join(newPath, part.FileName())) - - if err != nil { - httpErrorf(w, "error creating deb file: %s", err) - return - } - - defer dst.Close() - - if _, err := io.Copy(dst, part); err != nil { - httpErrorf(w, "error writing deb file: %s", err) - return - } - - // Get package name, if it already exists remove the old file. - f, err := newPackageFile(newPath, part.FileName()) - - if err != nil { - httpErrorf(w, "error loading package info: %s", err) - } - - if p, exists := packages[f.Info.Package]; exists { - v1, err1 := semver.Parse(p.Info.Version) - v2, err2 := semver.Parse(f.Info.Version) - - if err1 == nil && err2 == nil && v1.Compare(v2) > 0 && !force { - // Don't replace newer package - httpErrorf(w, "version in old package is greater than new: %s, %s - override with \"force\"", p.Info.Version, f.Info.Version) - return - } - - // Archive old file - log.Println("Replacing", p.Name, "with", f.Name) - - if err := os.Remove(p.Path); err != nil { - httpErrorf(w, "Unable to remove old package: %s", err) - return - } - } - - packages[f.Info.Package] = f - } - - log.Println("got lock, updating package list...") - - // Recreate the package index and release file. - - if err := createPackagesCached(config, distroName, archType, packages); err != nil { - httpErrorf(w, "error creating package: %s", err) - return - } - - err = createRelease(config, distroName, archType) - - if err != nil { - httpErrorf(w, "error creating package: %s", err) - return - } - - w.WriteHeader(http.StatusOK) - }) + distros[distroName] = distro } -func deleteHandler(config Conf) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != "DELETE" { - http.Error(w, "method not supported", http.StatusMethodNotAllowed) - return - } - var req DeleteObj +func uploadHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "method not supported", http.StatusMethodNotAllowed) + return + } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - httpErrorf(w, "failed to decode json: %s", err) - return - } + archType := r.URL.Query().Get("arch") - mutex.RLock() + if archType == "" { + archType = "all" + } - distro, exists := distros[req.DistroName] + distroName := r.URL.Query().Get("distro") - if !exists { - httpErrorf(w, "invalid distro: %s", req.DistroName) - mutex.RUnlock() - return - } + if distroName == "" { + distroName = "stable" + } - packages, exists := distro.Architectures[req.Arch] + key := r.URL.Query().Get("key") - if !exists { - httpErrorf(w, "invalid arch: %s", req.Arch) - mutex.RUnlock() - return - } + if key == "" || key != conf.Http.Key { + http.Error(w, "unauthorized", 403) + return + } + force := false + + forceStr := r.URL.Query().Get("force") + + if forceStr != "" && forceStr == "true" { + force = true + } + + reader, err := r.MultipartReader() + + if err != nil { + httpErrorf(w, "error creating multipart reader: %s", err) + return + } + + mutex.RLock() + + distro, exists := distros[distroName] + + if !exists { + httpErrorf(w, "invalid distro: %s", distroName) mutex.RUnlock() + return + } - debPath := filepath.Join(config.ArchPath(req.DistroName, req.Arch), req.Filename) + packages, exists := distro.Architectures[archType] - if err := os.Remove(debPath); err != nil { - httpErrorf(w, "failed to delete: %s", err) + if !exists { + httpErrorf(w, "invalid arch: %s", archType) + mutex.RUnlock() + return + } + + mutex.RUnlock() + + // Lock to prevent concurrent modification + mutex.Lock() + defer mutex.Unlock() + + for { + part, err := reader.NextPart() + + if err == io.EOF { + break + } + + if part.FileName() == "" { + continue + } + newPath := conf.PoolPackagePath(distroName, archType, part.FileName()) + + if _, err := os.Stat(newPath); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(newPath, 0755); err != nil { + httpErrorf(w, "error creating path: %s", err) + return + } + } + } + + dst, err := os.Create(filepath.Join(newPath, part.FileName())) + + if err != nil { + httpErrorf(w, "error creating deb file: %s", err) return } - key := r.URL.Query().Get("key") + defer dst.Close() - if key == "" || key != config.Key { - http.Error(w, "unauthorized", http.StatusForbidden) - return - } - mutex.Lock() - defer mutex.Unlock() - - log.Println("got lock, updating package list...") - if err := createPackagesCached(config, req.DistroName, req.Arch, packages); err != nil { - httpErrorf(w, "failed to delete package: %s", err) + if _, err := io.Copy(dst, part); err != nil { + httpErrorf(w, "error writing deb file: %s", err) return } - if err := createRelease(config, req.DistroName, req.Arch); err != nil { - httpErrorf(w, "failed to delete package: %s", err) - return + // Get package name, if it already exists remove the old file. + f, err := newPackageFile(newPath, part.FileName()) + + if err != nil { + httpErrorf(w, "error loading package info: %s", err) } - }) + + if p, exists := packages[f.Info.Package]; exists { + v1, err1 := semver.Parse(p.Info.Version) + v2, err2 := semver.Parse(f.Info.Version) + + if err1 == nil && err2 == nil && v1.Compare(v2) > 0 && !force { + // Don't replace newer package + httpErrorf(w, "version in old package is greater than new: %s, %s - override with \"force\"", p.Info.Version, f.Info.Version) + return + } + + // Archive old file + log.Println("Replacing", p.Name, "with", f.Name) + + if err := os.Remove(p.Path); err != nil { + httpErrorf(w, "Unable to remove old package: %s", err) + return + } + } + + packages[f.Info.Package] = f + } + + log.Println("got lock, updating package list...") + + // Recreate the package index and release file. + + if err := createPackagesCached(conf, distroName, archType, packages); err != nil { + httpErrorf(w, "error creating package: %s", err) + return + } + + err = createRelease(conf, distroName, archType) + + if err != nil { + httpErrorf(w, "error creating package: %s", err) + return + } + + w.WriteHeader(http.StatusCreated) +} + +func deleteHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != "DELETE" { + http.Error(w, "method not supported", http.StatusMethodNotAllowed) + return + } + var req DeleteObj + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httpErrorf(w, "failed to decode json: %s", err) + return + } + + mutex.RLock() + + distro, exists := distros[req.DistroName] + + if !exists { + httpErrorf(w, "invalid distro: %s", req.DistroName) + mutex.RUnlock() + return + } + + packages, exists := distro.Architectures[req.Arch] + + if !exists { + httpErrorf(w, "invalid arch: %s", req.Arch) + mutex.RUnlock() + return + } + + mutex.RUnlock() + + key := r.URL.Query().Get("key") + + if key == "" || key != conf.Http.Key { + http.Error(w, "unauthorized", http.StatusForbidden) + return + } + + debPath := filepath.Join(conf.ArchPath(req.DistroName, req.Arch), req.Filename) + + if err := os.Remove(debPath); err != nil { + httpErrorf(w, "failed to delete: %s", err) + return + } + + mutex.Lock() + defer mutex.Unlock() + + if err := createPackagesCached(conf, req.DistroName, req.Arch, packages); err != nil { + httpErrorf(w, "failed to delete package: %s", err) + return + } + + if err := createRelease(conf, req.DistroName, req.Arch); err != nil { + httpErrorf(w, "failed to delete package: %s", err) + return + } + + w.WriteHeader(http.StatusOK) } func httpErrorf(w http.ResponseWriter, format string, a ...interface{}) { diff --git a/src/meow.tf/deb-simple/packages.go b/src/meow.tf/deb-simple/packages.go index 91d7afa..9fd37d4 100644 --- a/src/meow.tf/deb-simple/packages.go +++ b/src/meow.tf/deb-simple/packages.go @@ -16,6 +16,7 @@ import ( "crypto/sha1" "crypto/sha256" "github.com/blang/semver" + "encoding/json" ) var ( @@ -23,20 +24,20 @@ var ( ) type PackageFile struct { - Path string - Name string - Size int64 - MD5Hash string - SHA1Hash string - SHA256Hash string - ControlData string - Info *Package + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + MD5Hash string `json:"md5"` + SHA1Hash string `json:"sha1"` + SHA256Hash string `json:"sha256"` + ControlData string `json:"control"` + Info *Package `json:"info"` } type Package struct { - Package string - Version string - Architecture string + Package string `json:"package"` + Version string `json:"version"` + Architecture string `json:"architecture"` } func parsePackageData(ctlData string) *Package { @@ -57,14 +58,50 @@ func parsePackageData(ctlData string) *Package { } type Distro struct { - Name string - Architectures map[string]map[string]*PackageFile + Name string `json:"name"` + Architectures map[string]map[string]*PackageFile `json:"architectures"` } var ( distros map[string]*Distro = make(map[string]*Distro) ) +func loadCache(dist string) error { + f, err := os.Open(filepath.Join(conf.DistPath(dist), "dist.json")) + + if err != nil { + return err + } + + defer f.Close() + + var distro Distro + + if err := json.NewDecoder(f).Decode(&distro); err != nil { + return err + } + + distros[dist] = &distro + + return nil +} + +func saveCache(dist *Distro) error { + f, err := os.Create(filepath.Join(conf.DistPath(dist.Name), "dist.json")) + + if err != nil { + return err + } + + defer f.Close() + + if err := json.NewEncoder(f).Encode(dist); err != nil { + return err + } + + return nil +} + func buildPackageList(config Conf, distro, arch string) (map[string]*PackageFile, error) { m := make(map[string]*PackageFile) diff --git a/src/meow.tf/deb-simple/server.go b/src/meow.tf/deb-simple/server.go index 7d037e1..8cb28c8 100644 --- a/src/meow.tf/deb-simple/server.go +++ b/src/meow.tf/deb-simple/server.go @@ -1,53 +1,22 @@ package simple import ( - "encoding/json" "flag" "fmt" "io/ioutil" "log" "net/http" "os" - "path/filepath" "sync" "strings" "golang.org/x/crypto/openpgp" "runtime" "errors" + "github.com/go-ini/ini" ) var VERSION string = "1.1.0" -type Conf struct { - ListenPort string `json:"listenPort"` - RootRepoPath string `json:"rootRepoPath"` - SupportArch []string `json:"supportedArch"` - DistroNames []string `json:"distroNames"` - PGPSecretKey string `json:"pgpSecretKey"` - PGPPassphrase string `json:"pgpPassphrase"` - EnableSSL bool `json:"enableSSL"` - SSLCert string `json:"SSLcert"` - SSLKey string `json:"SSLkey"` - Key string `json:"key"` -} - -func (c Conf) DistPath(distro string) string { - return filepath.Join(c.RootRepoPath, "dists", distro) -} - -func (c Conf) ArchPath(distro, arch string) string { - return filepath.Join(c.RootRepoPath, "dists", distro, "main/binary-"+arch) -} - -func (c Conf) PoolPath(distro, arch string) string { - return filepath.Join(c.RootRepoPath, "pool/main", distro, arch) -} - -func (c Conf) PoolPackagePath(distro, arch, name string) string { - name = packageName(name) - return filepath.Join(c.RootRepoPath, "pool/main", distro, arch, name[0:1], name) -} - func packageName(name string) string { if index := strings.Index(name, "_"); index != -1 { name = name[:index] @@ -63,9 +32,9 @@ type DeleteObj struct { var ( mutex sync.RWMutex - configFile = flag.String("c", "conf.json", "config file location") + configFile = flag.String("c", "conf.ini", "config file location") flagShowVersion = flag.Bool("version", false, "Show dnsconfig version") - parsedConfig = Conf{} + conf = Conf{} pgpEntity *openpgp.Entity ) @@ -83,45 +52,53 @@ func Start() { log.Fatalln("unable to read config file, exiting...") } - if err := json.Unmarshal(file, &parsedConfig); err != nil { - log.Fatalln("unable to marshal config file, exiting...") + if err := ini.MapTo(&conf, file); err != nil { + log.Fatalln("unable to marshal config file, exiting...", err) } - if err := createDirs(parsedConfig); err != nil { + if err := createDirs(conf); err != nil { log.Println(err) log.Fatalln("error creating directory structure, exiting") } - if err := setupPgp(parsedConfig); err != nil { + if err := setupPgp(conf); err != nil { log.Println(err) log.Fatalln("error loading pgp key, exiting") } log.Println("Indexing packages...") - for _, dist := range parsedConfig.DistroNames { - distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)} + for _, dist := range conf.Repo.DistroNames() { + if err := loadCache(dist); err != nil { + log.Println("Unable to load cached data for", dist, "- reindexing") + distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)} - go scanInitialPackages(parsedConfig, distro) + go scanInitialPackages(conf, distro) - distros[dist] = distro + distros[dist] = distro + } } - http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(parsedConfig.RootRepoPath)))) - http.Handle("/upload", uploadHandler(parsedConfig)) - http.Handle("/delete", deleteHandler(parsedConfig)) + mux := http.NewServeMux() - if parsedConfig.EnableSSL { + mux.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(conf.Repo.Root)))) + mux.HandleFunc("/rescan", rescanHandler) + mux.HandleFunc("/upload", uploadHandler) + mux.HandleFunc("/delete", deleteHandler) + + bind := fmt.Sprintf(":%d", conf.Http.Port) + + if conf.Http.SSL { log.Println("running with SSL enabled") - log.Fatalln(http.ListenAndServeTLS(":"+parsedConfig.ListenPort, parsedConfig.SSLCert, parsedConfig.SSLKey, nil)) + log.Fatalln(http.ListenAndServeTLS(bind, conf.Http.SSLCert, conf.Http.SSLKey, mux)) } else { log.Println("running without SSL enabled") - log.Fatalln(http.ListenAndServe(":"+parsedConfig.ListenPort, nil)) + log.Fatalln(http.ListenAndServe(bind, mux)) } } func scanInitialPackages(config Conf, dist *Distro) { - for _, arch := range config.SupportArch { + for _, arch := range config.Repo.ArchitectureNames() { files, err := buildPackageList(config, dist.Name, arch) if err != nil { @@ -134,14 +111,16 @@ func scanInitialPackages(config Conf, dist *Distro) { createPackagesCached(config, dist.Name, arch, files) createRelease(config, dist.Name, arch) } + + saveCache(dist) } func setupPgp(config Conf) error { - if config.PGPSecretKey == "" { + if config.PGP.Key == "" { return nil } - secretKey, err := os.Open(config.PGPSecretKey) + secretKey, err := os.Open(config.PGP.Key) if err != nil { return fmt.Errorf("failed to open private key ring file: %s", err) } @@ -159,7 +138,7 @@ func setupPgp(config Conf) error { pgpEntity = entitylist[0] - passphrase := []byte(config.PGPPassphrase) + passphrase := []byte(config.PGP.Passphrase) if err := pgpEntity.PrivateKey.Decrypt(passphrase); err != nil { return err @@ -177,8 +156,8 @@ func setupPgp(config Conf) error { } func createDirs(config Conf) error { - for _, distro := range config.DistroNames { - for _, arch := range config.SupportArch { + for _, distro := range config.Repo.DistroNames() { + for _, arch := range config.Repo.ArchitectureNames() { if _, err := os.Stat(config.ArchPath(distro, arch)); err != nil { if os.IsNotExist(err) { log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)