tyler/deb-simple
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Caching of packages, new config format

Browse Source
This commit is contained in:
Tyler 2017-06-11 20:41:49 -04:00
parent 1b7951fb75
commit bed2ffbec7
8 changed files with 360 additions and 265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,5 +1,6 @@
/deb-simple
/simple
/repo
vendor
conf.json
conf.ini
*.pprof

25
sample_conf.ini
View File

@ -1,12 +1,13 @@
{
"listenPort" : "9090",
"rootRepoPath" : "/opt/deb-simple/repo",
"supportedArch" : ["all","i386","amd64"],
"distroNames":["stable"],
"pgpSecretKey": "secring.gpg",
"pgpPassphrase" : "",
"enableSSL" : false,
"SSLcert" : "server.crt",
"SSLkey" : "server.key",
"key" : "abcdefg"
}
[http]
port = 9090
key = "abcdefg"
ssl = false
[repo]
root = "repo/"
distros = "stable"
architectures = "all,i386,amd64"
[pgp]
key = ""
passphrase = ""

55
src/meow.tf/deb-simple/config.go
View File

@ -1 +1,56 @@
package simple
import (
"path/filepath"
"strings"
)
type Conf struct {
Http HttpConf `ini:"http"`
Repo RepoConf `ini:"repo"`
PGP PGPConf `ini:"pgp"`
}
type HttpConf struct {
Port int `ini:"port"`
Key string `ini:"key"`
SSL bool `ini:"ssl"`
SSLCert string `ini:"cert"`
SSLKey string `ini:"key"`
}
type RepoConf struct {
Root string `ini:"root"`
Distros string `ini:"distros"`
Architectures string `ini:"architectures"`
}
type PGPConf struct {
Key string `ini:"key"`
Passphrase string `ini:"passphrase"`
}
func (c Conf) DistPath(distro string) string {
return filepath.Join(c.Repo.Root, "dists", distro)
}
func (c Conf) ArchPath(distro, arch string) string {
return filepath.Join(c.Repo.Root, "dists", distro, "main/binary-"+arch)
}
func (c Conf) PoolPath(distro, arch string) string {
return filepath.Join(c.Repo.Root, "pool/main", distro, arch)
}
func (c Conf) PoolPackagePath(distro, arch, name string) string {
name = packageName(name)
return filepath.Join(c.Repo.Root, "pool/main", distro, arch, name[0:1], name)
}
func (c RepoConf) DistroNames() []string {
return strings.Split(c.Distros, ",")
}
func (c RepoConf) ArchitectureNames() []string {
return strings.Split(c.Architectures, ",")
}

6
src/meow.tf/deb-simple/glide.lock generated
View File

@ -1,10 +1,12 @@
hash: 79d0f0e99e99ce9318bb0e8c6ab398657a20679dbe3c421c6fc588db7a738131
updated: 2017-06-11T03:04:15.102617-04:00
hash: 33760adb1090508f6d56dfc1d600a664790a280e5fc4d84cb26a7e11ad0f1f35
updated: 2017-06-11T20:13:27.6668377-04:00
imports:
- name: github.com/blakesmith/ar
version: 8bd4349a67f2533b078dbc524689d15dba0f4659
- name: github.com/blang/semver
version: b38d23b8782a487059e8fc8773e9a5b228a77cb6
- name: github.com/go-ini/ini
version: d3de07a94d22b4a0972deb4b96d790c2c0ce8333
- name: golang.org/x/crypto
version: 7e9105388ebff089b3f99f0ef676ea55a6da3a7e
subpackages:

2
src/meow.tf/deb-simple/glide.yaml
View File

@ -6,3 +6,5 @@ import:
- openpgp
- package: github.com/blang/semver
version: ^3.5.0
- package: github.com/go-ini/ini
version: ^1.28.0

384
src/meow.tf/deb-simple/http.go
View File

@ -11,214 +11,232 @@ import (
"github.com/blang/semver"
)
func uploadHandler(config Conf) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "method not supported", http.StatusMethodNotAllowed)
return
}
archType := r.URL.Query().Get("arch")
func rescanHandler(w http.ResponseWriter, r *http.Request) {
distroName := r.URL.Query().Get("distro")
if archType == "" {
archType = "all"
}
if distroName == "" {
distroName = "stable"
}
distroName := r.URL.Query().Get("distro")
if _, exists := distros[distroName]; !exists {
httpErrorf(w, "Unable to find distro %s", distroName)
return
}
if distroName == "" {
distroName = "stable"
}
distro := &Distro{Name: distroName, Architectures: make(map[string]map[string]*PackageFile)}
key := r.URL.Query().Get("key")
scanInitialPackages(conf, distro)
if key == "" || key != config.Key {
http.Error(w, "unauthorized", 403)
return
}
force := false
forceStr := r.URL.Query().Get("force")
if forceStr != "" && forceStr == "true" {
force = true
}
reader, err := r.MultipartReader()
if err != nil {
httpErrorf(w, "error creating multipart reader: %s", err)
return
}
mutex.RLock()
distro, exists := distros[distroName]
if !exists {
httpErrorf(w, "invalid distro: %s", distroName)
mutex.RUnlock()
return
}
packages, exists := distro.Architectures[archType]
if !exists {
httpErrorf(w, "invalid arch: %s", archType)
mutex.RUnlock()
return
}
mutex.RUnlock()
// Lock to prevent concurrent modification
mutex.Lock()
defer mutex.Unlock()
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if part.FileName() == "" {
continue
}
newPath := config.PoolPackagePath(distroName, archType, part.FileName())
if _, err := os.Stat(newPath); err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(newPath, 0755); err != nil {
httpErrorf(w, "error creating path: %s", err)
return
}
}
}
dst, err := os.Create(filepath.Join(newPath, part.FileName()))
if err != nil {
httpErrorf(w, "error creating deb file: %s", err)
return
}
defer dst.Close()
if _, err := io.Copy(dst, part); err != nil {
httpErrorf(w, "error writing deb file: %s", err)
return
}
// Get package name, if it already exists remove the old file.
f, err := newPackageFile(newPath, part.FileName())
if err != nil {
httpErrorf(w, "error loading package info: %s", err)
}
if p, exists := packages[f.Info.Package]; exists {
v1, err1 := semver.Parse(p.Info.Version)
v2, err2 := semver.Parse(f.Info.Version)
if err1 == nil && err2 == nil && v1.Compare(v2) > 0 && !force {
// Don't replace newer package
httpErrorf(w, "version in old package is greater than new: %s, %s - override with \"force\"", p.Info.Version, f.Info.Version)
return
}
// Archive old file
log.Println("Replacing", p.Name, "with", f.Name)
if err := os.Remove(p.Path); err != nil {
httpErrorf(w, "Unable to remove old package: %s", err)
return
}
}
packages[f.Info.Package] = f
}
log.Println("got lock, updating package list...")
// Recreate the package index and release file.
if err := createPackagesCached(config, distroName, archType, packages); err != nil {
httpErrorf(w, "error creating package: %s", err)
return
}
err = createRelease(config, distroName, archType)
if err != nil {
httpErrorf(w, "error creating package: %s", err)
return
}
w.WriteHeader(http.StatusOK)
})
distros[distroName] = distro
}
func deleteHandler(config Conf) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "DELETE" {
http.Error(w, "method not supported", http.StatusMethodNotAllowed)
return
}
var req DeleteObj
func uploadHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "method not supported", http.StatusMethodNotAllowed)
return
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
httpErrorf(w, "failed to decode json: %s", err)
return
}
archType := r.URL.Query().Get("arch")
mutex.RLock()
if archType == "" {
archType = "all"
}
distro, exists := distros[req.DistroName]
distroName := r.URL.Query().Get("distro")
if !exists {
httpErrorf(w, "invalid distro: %s", req.DistroName)
mutex.RUnlock()
return
}
if distroName == "" {
distroName = "stable"
}
packages, exists := distro.Architectures[req.Arch]
key := r.URL.Query().Get("key")
if !exists {
httpErrorf(w, "invalid arch: %s", req.Arch)
mutex.RUnlock()
return
}
if key == "" || key != conf.Http.Key {
http.Error(w, "unauthorized", 403)
return
}
force := false
forceStr := r.URL.Query().Get("force")
if forceStr != "" && forceStr == "true" {
force = true
}
reader, err := r.MultipartReader()
if err != nil {
httpErrorf(w, "error creating multipart reader: %s", err)
return
}
mutex.RLock()
distro, exists := distros[distroName]
if !exists {
httpErrorf(w, "invalid distro: %s", distroName)
mutex.RUnlock()
return
}
debPath := filepath.Join(config.ArchPath(req.DistroName, req.Arch), req.Filename)
packages, exists := distro.Architectures[archType]
if err := os.Remove(debPath); err != nil {
httpErrorf(w, "failed to delete: %s", err)
if !exists {
httpErrorf(w, "invalid arch: %s", archType)
mutex.RUnlock()
return
}
mutex.RUnlock()
// Lock to prevent concurrent modification
mutex.Lock()
defer mutex.Unlock()
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if part.FileName() == "" {
continue
}
newPath := conf.PoolPackagePath(distroName, archType, part.FileName())
if _, err := os.Stat(newPath); err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(newPath, 0755); err != nil {
httpErrorf(w, "error creating path: %s", err)
return
}
}
}
dst, err := os.Create(filepath.Join(newPath, part.FileName()))
if err != nil {
httpErrorf(w, "error creating deb file: %s", err)
return
}
key := r.URL.Query().Get("key")
defer dst.Close()
if key == "" || key != config.Key {
http.Error(w, "unauthorized", http.StatusForbidden)
return
}
mutex.Lock()
defer mutex.Unlock()
log.Println("got lock, updating package list...")
if err := createPackagesCached(config, req.DistroName, req.Arch, packages); err != nil {
httpErrorf(w, "failed to delete package: %s", err)
if _, err := io.Copy(dst, part); err != nil {
httpErrorf(w, "error writing deb file: %s", err)
return
}
if err := createRelease(config, req.DistroName, req.Arch); err != nil {
httpErrorf(w, "failed to delete package: %s", err)
return
// Get package name, if it already exists remove the old file.
f, err := newPackageFile(newPath, part.FileName())
if err != nil {
httpErrorf(w, "error loading package info: %s", err)
}
})
if p, exists := packages[f.Info.Package]; exists {
v1, err1 := semver.Parse(p.Info.Version)
v2, err2 := semver.Parse(f.Info.Version)
if err1 == nil && err2 == nil && v1.Compare(v2) > 0 && !force {
// Don't replace newer package
httpErrorf(w, "version in old package is greater than new: %s, %s - override with \"force\"", p.Info.Version, f.Info.Version)
return
}
// Archive old file
log.Println("Replacing", p.Name, "with", f.Name)
if err := os.Remove(p.Path); err != nil {
httpErrorf(w, "Unable to remove old package: %s", err)
return
}
}
packages[f.Info.Package] = f
}
log.Println("got lock, updating package list...")
// Recreate the package index and release file.
if err := createPackagesCached(conf, distroName, archType, packages); err != nil {
httpErrorf(w, "error creating package: %s", err)
return
}
err = createRelease(conf, distroName, archType)
if err != nil {
httpErrorf(w, "error creating package: %s", err)
return
}
w.WriteHeader(http.StatusCreated)
}
func deleteHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != "DELETE" {
http.Error(w, "method not supported", http.StatusMethodNotAllowed)
return
}
var req DeleteObj
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
httpErrorf(w, "failed to decode json: %s", err)
return
}
mutex.RLock()
distro, exists := distros[req.DistroName]
if !exists {
httpErrorf(w, "invalid distro: %s", req.DistroName)
mutex.RUnlock()
return
}
packages, exists := distro.Architectures[req.Arch]
if !exists {
httpErrorf(w, "invalid arch: %s", req.Arch)
mutex.RUnlock()
return
}
mutex.RUnlock()
key := r.URL.Query().Get("key")
if key == "" || key != conf.Http.Key {
http.Error(w, "unauthorized", http.StatusForbidden)
return
}
debPath := filepath.Join(conf.ArchPath(req.DistroName, req.Arch), req.Filename)
if err := os.Remove(debPath); err != nil {
httpErrorf(w, "failed to delete: %s", err)
return
}
mutex.Lock()
defer mutex.Unlock()
if err := createPackagesCached(conf, req.DistroName, req.Arch, packages); err != nil {
httpErrorf(w, "failed to delete package: %s", err)
return
}
if err := createRelease(conf, req.DistroName, req.Arch); err != nil {
httpErrorf(w, "failed to delete package: %s", err)
return
}
w.WriteHeader(http.StatusOK)
}
func httpErrorf(w http.ResponseWriter, format string, a ...interface{}) {

63
src/meow.tf/deb-simple/packages.go
View File

@ -16,6 +16,7 @@ import (
"crypto/sha1"
"crypto/sha256"
"github.com/blang/semver"
"encoding/json"
)
var (
@ -23,20 +24,20 @@ var (
)
type PackageFile struct {
Path string
Name string
Size int64
MD5Hash string
SHA1Hash string
SHA256Hash string
ControlData string
Info *Package
Path string `json:"path"`
Name string `json:"name"`
Size int64 `json:"size"`
MD5Hash string `json:"md5"`
SHA1Hash string `json:"sha1"`
SHA256Hash string `json:"sha256"`
ControlData string `json:"control"`
Info *Package `json:"info"`
}
type Package struct {
Package string
Version string
Architecture string
Package string `json:"package"`
Version string `json:"version"`
Architecture string `json:"architecture"`
}
func parsePackageData(ctlData string) *Package {
@ -57,14 +58,50 @@ func parsePackageData(ctlData string) *Package {
}
type Distro struct {
Name string
Architectures map[string]map[string]*PackageFile
Name string `json:"name"`
Architectures map[string]map[string]*PackageFile `json:"architectures"`
}
var (
distros map[string]*Distro = make(map[string]*Distro)
)
func loadCache(dist string) error {
f, err := os.Open(filepath.Join(conf.DistPath(dist), "dist.json"))
if err != nil {
return err
}
defer f.Close()
var distro Distro
if err := json.NewDecoder(f).Decode(&distro); err != nil {
return err
}
distros[dist] = &distro
return nil
}
func saveCache(dist *Distro) error {
f, err := os.Create(filepath.Join(conf.DistPath(dist.Name), "dist.json"))
if err != nil {
return err
}
defer f.Close()
if err := json.NewEncoder(f).Encode(dist); err != nil {
return err
}
return nil
}
func buildPackageList(config Conf, distro, arch string) (map[string]*PackageFile, error) {
m := make(map[string]*PackageFile)

87
src/meow.tf/deb-simple/server.go
View File

@ -1,53 +1,22 @@
package simple
import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"sync"
"strings"
"golang.org/x/crypto/openpgp"
"runtime"
"errors"
"github.com/go-ini/ini"
)
var VERSION string = "1.1.0"
type Conf struct {
ListenPort string `json:"listenPort"`
RootRepoPath string `json:"rootRepoPath"`
SupportArch []string `json:"supportedArch"`
DistroNames []string `json:"distroNames"`
PGPSecretKey string `json:"pgpSecretKey"`
PGPPassphrase string `json:"pgpPassphrase"`
EnableSSL bool `json:"enableSSL"`
SSLCert string `json:"SSLcert"`
SSLKey string `json:"SSLkey"`
Key string `json:"key"`
}
func (c Conf) DistPath(distro string) string {
return filepath.Join(c.RootRepoPath, "dists", distro)
}
func (c Conf) ArchPath(distro, arch string) string {
return filepath.Join(c.RootRepoPath, "dists", distro, "main/binary-"+arch)
}
func (c Conf) PoolPath(distro, arch string) string {
return filepath.Join(c.RootRepoPath, "pool/main", distro, arch)
}
func (c Conf) PoolPackagePath(distro, arch, name string) string {
name = packageName(name)
return filepath.Join(c.RootRepoPath, "pool/main", distro, arch, name[0:1], name)
}
func packageName(name string) string {
if index := strings.Index(name, "_"); index != -1 {
name = name[:index]
@ -63,9 +32,9 @@ type DeleteObj struct {
var (
mutex sync.RWMutex
configFile = flag.String("c", "conf.json", "config file location")
configFile = flag.String("c", "conf.ini", "config file location")
flagShowVersion = flag.Bool("version", false, "Show dnsconfig version")
parsedConfig = Conf{}
conf = Conf{}
pgpEntity *openpgp.Entity
)
@ -83,45 +52,53 @@ func Start() {
log.Fatalln("unable to read config file, exiting...")
}
if err := json.Unmarshal(file, &parsedConfig); err != nil {
log.Fatalln("unable to marshal config file, exiting...")
if err := ini.MapTo(&conf, file); err != nil {
log.Fatalln("unable to marshal config file, exiting...", err)
}
if err := createDirs(parsedConfig); err != nil {
if err := createDirs(conf); err != nil {
log.Println(err)
log.Fatalln("error creating directory structure, exiting")
}
if err := setupPgp(parsedConfig); err != nil {
if err := setupPgp(conf); err != nil {
log.Println(err)
log.Fatalln("error loading pgp key, exiting")
}
log.Println("Indexing packages...")
for _, dist := range parsedConfig.DistroNames {
distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)}
for _, dist := range conf.Repo.DistroNames() {
if err := loadCache(dist); err != nil {
log.Println("Unable to load cached data for", dist, "- reindexing")
distro := &Distro{Name: dist, Architectures: make(map[string]map[string]*PackageFile)}
go scanInitialPackages(parsedConfig, distro)
go scanInitialPackages(conf, distro)
distros[dist] = distro
distros[dist] = distro
}
}
http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(parsedConfig.RootRepoPath))))
http.Handle("/upload", uploadHandler(parsedConfig))
http.Handle("/delete", deleteHandler(parsedConfig))
mux := http.NewServeMux()
if parsedConfig.EnableSSL {
mux.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(conf.Repo.Root))))
mux.HandleFunc("/rescan", rescanHandler)
mux.HandleFunc("/upload", uploadHandler)
mux.HandleFunc("/delete", deleteHandler)
bind := fmt.Sprintf(":%d", conf.Http.Port)
if conf.Http.SSL {
log.Println("running with SSL enabled")
log.Fatalln(http.ListenAndServeTLS(":"+parsedConfig.ListenPort, parsedConfig.SSLCert, parsedConfig.SSLKey, nil))
log.Fatalln(http.ListenAndServeTLS(bind, conf.Http.SSLCert, conf.Http.SSLKey, mux))
} else {
log.Println("running without SSL enabled")
log.Fatalln(http.ListenAndServe(":"+parsedConfig.ListenPort, nil))
log.Fatalln(http.ListenAndServe(bind, mux))
}
}
func scanInitialPackages(config Conf, dist *Distro) {
for _, arch := range config.SupportArch {
for _, arch := range config.Repo.ArchitectureNames() {
files, err := buildPackageList(config, dist.Name, arch)
if err != nil {
@ -134,14 +111,16 @@ func scanInitialPackages(config Conf, dist *Distro) {
createPackagesCached(config, dist.Name, arch, files)
createRelease(config, dist.Name, arch)
}
saveCache(dist)
}
func setupPgp(config Conf) error {
if config.PGPSecretKey == "" {
if config.PGP.Key == "" {
return nil
}
secretKey, err := os.Open(config.PGPSecretKey)
secretKey, err := os.Open(config.PGP.Key)
if err != nil {
return fmt.Errorf("failed to open private key ring file: %s", err)
}
@ -159,7 +138,7 @@ func setupPgp(config Conf) error {
pgpEntity = entitylist[0]
passphrase := []byte(config.PGPPassphrase)
passphrase := []byte(config.PGP.Passphrase)
if err := pgpEntity.PrivateKey.Decrypt(passphrase); err != nil {
return err
@ -177,8 +156,8 @@ func setupPgp(config Conf) error {
}
func createDirs(config Conf) error {
for _, distro := range config.DistroNames {
for _, arch := range config.SupportArch {
for _, distro := range config.Repo.DistroNames() {
for _, arch := range config.Repo.ArchitectureNames() {
if _, err := os.Stat(config.ArchPath(distro, arch)); err != nil {
if os.IsNotExist(err) {
log.Printf("Directory for %s (%s) does not exist, creating", distro, arch)