From a0d42bbbc3320780b7d7cef328bc9b5b50071b58 Mon Sep 17 00:00:00 2001 From: Tyler Date: Thu, 13 Jun 2019 20:01:05 -0400 Subject: [PATCH] Better caching, resolving some cache issues --- cache/cache.go | 34 +++++++++++++++++++++++++ cache/gcache.go | 47 ++++++++++++++++++++++++++++++++++ cache/memcached.go | 32 +++++++++++++++++++++++ cache/null.go | 14 +++++++++++ cache/redis.go | 29 +++++++++++++++++++++ go.mod | 2 ++ go.sum | 4 +++ main.go | 63 ++++++++++++++++++++++++---------------------- 8 files changed, 195 insertions(+), 30 deletions(-) create mode 100644 cache/cache.go create mode 100644 cache/gcache.go create mode 100644 cache/memcached.go create mode 100644 cache/null.go create mode 100644 cache/redis.go diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..02cbcaf --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,34 @@ +package cache + +import ( + "net/url" + "time" +) + +type Provider interface { + Get(key string) ([]byte, error) + Set(key string, b []byte, d time.Duration) error +} + +func ForURI(uri string) Provider { + u, err := url.Parse(uri) + + if err != nil { + return nil + } + + if u.Scheme == "" && u.Path != "" { + u.Scheme = u.Path + } + + switch u.Scheme { + case "memcached": + return NewMemcached(u) + case "redis": + return NewRedisCache(u) + case "gcache": + return NewGcache(u) + } + + return &NullCache{} +} diff --git a/cache/gcache.go b/cache/gcache.go new file mode 100644 index 0000000..f32d4dc --- /dev/null +++ b/cache/gcache.go @@ -0,0 +1,47 @@ +package cache + +import ( + "github.com/bluele/gcache" + "net/url" + "strconv" + "time" +) + +type Gcache struct { + cache gcache.Cache +} + +func NewGcache(u *url.URL) *Gcache { + size := 128 + + q := u.Query() + + if sizeStr := q.Get("size"); sizeStr != "" { + var err error + size, err = strconv.Atoi(sizeStr) + + if err != nil { + size = 128 + } + } + + gc := gcache.New(size). + LRU(). + Build() + + return &Gcache{cache: gc} +} + +func (c *Gcache) Get(key string) ([]byte, error) { + res, err := c.cache.Get(key) + + if err != nil { + return nil, err + } + + return res.([]byte), err +} + +func (c *Gcache) Set(key string, b []byte, d time.Duration) error { + return c.cache.SetWithExpire(key, b, d) +} diff --git a/cache/memcached.go b/cache/memcached.go new file mode 100644 index 0000000..aa903bf --- /dev/null +++ b/cache/memcached.go @@ -0,0 +1,32 @@ +package cache + +import ( + "github.com/bradfitz/gomemcache/memcache" + "net/url" + "strings" + "time" +) + +type Memcached struct { + client *memcache.Client +} + +func NewMemcached(u *url.URL) *Memcached { + mc := memcache.New(strings.Split(u.Host, ",")...) + + return &Memcached{client: mc} +} + +func (m *Memcached) Get(key string) ([]byte, error) { + item, err := m.client.Get(key) + + if err != nil { + return nil, err + } + + return item.Value, nil +} + +func (m *Memcached) Set(key string, b []byte, d time.Duration) error { + return m.client.Set(&memcache.Item{Key: key, Value: b, Expiration: int32(d.Seconds())}) +} diff --git a/cache/null.go b/cache/null.go new file mode 100644 index 0000000..07a50a1 --- /dev/null +++ b/cache/null.go @@ -0,0 +1,14 @@ +package cache + +import "time" + +type NullCache struct { +} + +func (n *NullCache) Get(key string) ([]byte, error) { + return nil, nil +} + +func (n *NullCache) Set(key string, b []byte, d time.Duration) error { + return nil +} diff --git a/cache/redis.go b/cache/redis.go new file mode 100644 index 0000000..2c4e6c2 --- /dev/null +++ b/cache/redis.go @@ -0,0 +1,29 @@ +package cache + +import ( + "github.com/go-redis/redis" + "net/url" + "time" +) + +type RedisCache struct { + client *redis.Client +} + +func NewRedisCache(uri *url.URL) *RedisCache { + client := redis.NewClient(&redis.Options{ + Addr: uri.Host, + Password: "", // no password set + DB: 0, // use default DB + }) + + return &RedisCache{client: client} +} + +func (c *RedisCache) Get(key string) ([]byte, error) { + return c.client.Get(key).Bytes() +} + +func (c *RedisCache) Set(key string, b []byte, d time.Duration) error { + return c.client.Set(key, b, d).Err() +} diff --git a/go.mod b/go.mod index 2adea7e..fb6b27c 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.12 require ( github.com/PuerkitoBio/goquery v1.5.1-0.20190109230704-3dcf72e6c17f + github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 + github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668 github.com/go-redis/redis v6.14.2+incompatible github.com/julienschmidt/httprouter v1.2.0 github.com/onsi/ginkgo v1.8.0 // indirect diff --git a/go.sum b/go.sum index 40faec3..36f12ec 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,10 @@ github.com/PuerkitoBio/goquery v1.5.1-0.20190109230704-3dcf72e6c17f h1:cWOyRTtBc github.com/PuerkitoBio/goquery v1.5.1-0.20190109230704-3dcf72e6c17f/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 h1:yCfXxYaelOyqnia8F/Yng47qhmfC9nKTRIbYRrRueq4= +github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833/go.mod h1:8c4/i2VlovMO2gBnHGQPN5EJw+H0lx1u/5p+cgsXtCk= +github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668 h1:U/lr3Dgy4WK+hNk4tyD+nuGjpVLPEHuJSFXMw11/HPA= +github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= diff --git a/main.go b/main.go index 29fe9f8..914fb01 100644 --- a/main.go +++ b/main.go @@ -1,11 +1,13 @@ package main import ( + "crypto/md5" + "encoding/hex" "encoding/json" "errors" "flag" + "git.meow.tf/ow-api/ow-api/cache" "git.meow.tf/ow-api/ow-api/json-patch" - "github.com/go-redis/redis" "github.com/julienschmidt/httprouter" "github.com/rs/cors" "log" @@ -16,7 +18,7 @@ import ( ) const ( - Version = "2.0.12" + Version = "2.1.0" OpAdd = "add" OpRemove = "remove" @@ -42,9 +44,13 @@ type awardsStats struct { } var ( - flagBind = flag.String("bind-address", ":8080", "Address to bind to for http requests") + flagBind = flag.String("bind-address", ":8080", "Address to bind to for http requests") + flagCache = flag.String("cache", "redis://localhost:6379", "Cache uri or 'none' to disable") + flagCacheTime = flag.Int("cacheTime", 600, "Cache time in seconds") - client *redis.Client + cacheProvider cache.Provider + + cacheTime time.Duration profilePatch *jsonpatch.Patch @@ -54,11 +60,9 @@ var ( func main() { loadHeroNames() - client = redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - Password: "", // no password set - DB: 0, // use default DB - }) + cacheProvider = cache.ForURI(*flagCache) + + cacheTime = time.Duration(*flagCacheTime) * time.Second var err error @@ -93,6 +97,7 @@ func main() { // Version router.GET("/v1/version", versionHandler) + router.HEAD("/v1/status", statusHandler) router.GET("/v1/status", statusHandler) c := cors.New(cors.Options{ @@ -140,7 +145,7 @@ func injectPlatform(platform string, handler httprouter.Handle) httprouter.Handl } } -func statsResponse(w http.ResponseWriter, ps httprouter.Params, patch *jsonpatch.Patch) ([]byte, error) { +func statsResponse(w http.ResponseWriter, ps httprouter.Params, patch *jsonpatch.Patch, cacheAddition string) ([]byte, error) { var stats *ovrstat.PlayerStats var err error @@ -150,6 +155,10 @@ func statsResponse(w http.ResponseWriter, ps httprouter.Params, patch *jsonpatch cacheKey := generateCacheKey(ps) + if cacheAddition != "" { + cacheKey += "-" + cacheAddition + } + if region := ps.ByName("region"); region != "" { stats, err = ovrstat.PCStats(tag) } else if platform := ps.ByName("platform"); platform != "" { @@ -164,7 +173,7 @@ func statsResponse(w http.ResponseWriter, ps httprouter.Params, patch *jsonpatch // Caching of full response for modification - res, err := client.Get(cacheKey).Bytes() + res, err := cacheProvider.Get(cacheKey) if res != nil && err == nil { if patch != nil { @@ -247,7 +256,9 @@ func statsResponse(w http.ResponseWriter, ps httprouter.Params, patch *jsonpatch } // Cache response - client.Set(cacheKey, b, 10*time.Minute) + if cacheTime > 0 { + cacheProvider.Set(cacheKey, b, cacheTime) + } if patch != nil { // Apply filter patch @@ -258,7 +269,7 @@ func statsResponse(w http.ResponseWriter, ps httprouter.Params, patch *jsonpatch } func stats(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - data, err := statsResponse(w, ps, nil) + data, err := statsResponse(w, ps, nil, "") if err != nil { writeError(w, err) @@ -271,26 +282,14 @@ func stats(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { } func profile(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - cacheKey := generateCacheKey(ps) + "-profile" - - // Check cache for -profile to prevent jsonpatch calls - res, err := client.Get(cacheKey).Bytes() - - if res != nil && err == nil { - w.Write(res) - return - } - // Cache result for profile specifically - data, err := statsResponse(w, ps, profilePatch) + data, err := statsResponse(w, ps, profilePatch, "profile") if err != nil { writeError(w, err) return } - client.Set(cacheKey, data, 10*time.Minute) - w.Header().Set("Content-Type", "application/json") w.Write(data) @@ -339,8 +338,10 @@ func heroes(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { return } + cacheKey := md5.New().Sum([]byte(strings.Join(names, ","))) + // Create a patch to remove all but specified heroes - data, err := statsResponse(w, ps, patch) + data, err := statsResponse(w, ps, patch, "heroes-"+hex.EncodeToString(cacheKey)) if err != nil { writeError(w, err) @@ -414,8 +415,10 @@ func statusHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) status.Error = err.Error() } - if err := json.NewEncoder(w).Encode(status); err != nil { - writeError(w, err) + if r.Method != http.MethodHead { + if err := json.NewEncoder(w).Encode(status); err != nil { + writeError(w, err) + } } } @@ -441,7 +444,7 @@ func generateCacheKey(ps httprouter.Params) string { tag := ps.ByName("tag") if region := ps.ByName("region"); region != "" { - cacheKey = "pc-" + region + "-" + tag + cacheKey = "pc-" + tag } else if platform := ps.ByName("platform"); platform != "" { cacheKey = platform + "-" + tag }