[full-ci] More cache cleanup (#6134)

* Make env var names consistent with the others

* Remove unused OCS cache

* Use the same cache database for shared caches

* Bump reva
This commit is contained in:
Andre Duffeck
2023-04-25 15:46:59 +02:00
committed by GitHub
parent 7b54a3595a
commit be569b7c59
15 changed files with 132 additions and 90 deletions
+2 -2
View File
@@ -13,7 +13,7 @@ require (
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/coreos/go-oidc/v3 v3.4.0
github.com/cs3org/go-cs3apis v0.0.0-20221012090518-ef2996678965
github.com/cs3org/reva/v2 v2.12.1-0.20230424091007-8d8b567179b1
github.com/cs3org/reva/v2 v2.12.1-0.20230425110812-8a3352fb1960
github.com/disintegration/imaging v1.6.2
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
github.com/egirna/icap-client v0.1.1
@@ -183,7 +183,7 @@ require (
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-micro/plugins/v4/store/nats-js v1.1.0 // indirect
github.com/go-micro/plugins/v4/store/redis v1.2.0 // indirect
github.com/go-micro/plugins/v4/store/redis v1.2.1-0.20230405210006-efd9191305c5 // indirect
github.com/go-redis/redis/v8 v8.11.5 // indirect
github.com/go-resty/resty/v2 v2.7.0 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
+4 -4
View File
@@ -627,8 +627,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo
github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4=
github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc=
github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA=
github.com/cs3org/reva/v2 v2.12.1-0.20230424091007-8d8b567179b1 h1:ds7JOiVqRoMR1EAkoxHJEEA8iX8JirO3bPxXACGE8iA=
github.com/cs3org/reva/v2 v2.12.1-0.20230424091007-8d8b567179b1/go.mod h1:2SKiycp0NwrLm/c1YeKUHwLSwPqV7hKfFEMdYgEaDxo=
github.com/cs3org/reva/v2 v2.12.1-0.20230425110812-8a3352fb1960 h1:QEQ9tu4S835dmRUryRj0JTQJPBg1AgPxXyVHrPtOY4U=
github.com/cs3org/reva/v2 v2.12.1-0.20230425110812-8a3352fb1960/go.mod h1:VxBmpOvIKlgKLPOsHun+fABopzX+3ZELPAp3N5bQMsM=
github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
@@ -792,8 +792,8 @@ github.com/go-micro/plugins/v4/server/http v1.2.1 h1:Cia924J90rgFT/4qWWvyLvN+XqE
github.com/go-micro/plugins/v4/server/http v1.2.1/go.mod h1:YuAjaSPxcn3LI8j2FUsqx0Rxunrj4YwDV41Ax76rLl0=
github.com/go-micro/plugins/v4/store/nats-js v1.1.0 h1:6Fe1/eLtg8kRyaGvMILp4olYtTDGwYNBXyb1sYfAWGk=
github.com/go-micro/plugins/v4/store/nats-js v1.1.0/go.mod h1:jJf7Gm39OafZlT3s3UE2/9NIYj6OlI2fmZ4czSA3gvo=
github.com/go-micro/plugins/v4/store/redis v1.2.0 h1:jR7sHOD1a735cxyBNFif58tP0Ck8OUklpDN1IzzDoRg=
github.com/go-micro/plugins/v4/store/redis v1.2.0/go.mod h1:MbCG0YiyPqETTtm7uHFmxQNCaW1o9hBoYtFwhbVjLUg=
github.com/go-micro/plugins/v4/store/redis v1.2.1-0.20230405210006-efd9191305c5 h1:W5HQDVr+6u9yQpnFZxMBRQd2rQCjOw/17Yb3dp6SQhE=
github.com/go-micro/plugins/v4/store/redis v1.2.1-0.20230405210006-efd9191305c5/go.mod h1:MbCG0YiyPqETTtm7uHFmxQNCaW1o9hBoYtFwhbVjLUg=
github.com/go-micro/plugins/v4/transport/grpc v1.1.0 h1:mXfDYfFQLnVDzjGY3o84oe4prfux9h8txsnA19dKsj8=
github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0 h1:EQj4l7fuOSz8ueUYhFlpZPp9+tN4JeONL32ARRKXW/U=
github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0/go.mod h1:JR9Ox/iJIrcXm8nCWdAEBsyG7Q7lyMLzsTZPfXrqvwo=
@@ -107,7 +107,7 @@ func DefaultConfig() *config.Config {
HomeNamespace: "/users/{{.Id.OpaqueId}}",
AdditionalInfoAttribute: "{{.Mail}}",
ResourceInfoCacheType: "memory",
ResourceInfoCacheDatabase: "frontend",
ResourceInfoCacheDatabase: "ocis",
ResourceInfoCacheTTL: 0,
},
Middleware: config.Middleware{
@@ -38,7 +38,7 @@ func DefaultConfig() *config.Config {
TransferExpires: 24 * 60 * 60,
Cache: config.Cache{
Store: "memory",
Database: "users",
Database: "ocis",
StatCacheTTL: 300,
ProviderCacheTTL: 300,
CreateHomeCacheTTL: 300,
-13
View File
@@ -1,13 +0,0 @@
package config
import "time"
// Cache defines the available configuration for the cache store
type Cache struct {
Store string `yaml:"store" env:"OCIS_CACHE_STORE;OCS_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;OCS_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
Database string `yaml:"database" env:"OCS_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
Table string `yaml:"table" env:"OCS_CACHE_STORE_TABLE" desc:"The database table the store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;OCS_CACHE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '336h' (2 weeks)."`
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;OCS_CACHE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured. Defaults to 512."`
}
-1
View File
@@ -14,7 +14,6 @@ type Config struct {
Tracing *Tracing `yaml:"tracing"`
Log *Log `yaml:"log"`
Cache *Cache `yaml:"cache"`
Debug Debug `yaml:"debug"`
HTTP HTTP `yaml:"http"`
@@ -66,16 +66,6 @@ func EnsureDefaults(cfg *config.Config) {
cfg.Tracing = &config.Tracing{}
}
if cfg.Cache == nil && cfg.Commons != nil && cfg.Commons.Cache != nil {
cfg.Cache = &config.Cache{
Store: cfg.Commons.Cache.Store,
Nodes: cfg.Commons.Cache.Nodes,
Size: cfg.Commons.Cache.Size,
}
} else if cfg.Cache == nil {
cfg.Cache = &config.Cache{}
}
if cfg.TokenManager == nil && cfg.Commons != nil && cfg.Commons.TokenManager != nil {
cfg.TokenManager = &config.TokenManager{
JWTSecret: cfg.Commons.TokenManager.JWTSecret,
+1 -1
View File
@@ -122,7 +122,7 @@ type JWKS struct {
// Cache is a TTL cache configuration.
type Cache struct {
Store string `yaml:"store" env:"OCIS_CACHE_STORE;PROXY_OIDC_USERINFO_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;PROXY_OIDC_USERINFO_CACHE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
Nodes []string `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;PROXY_OIDC_USERINFO_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
Table string `yaml:"table" env:"PROXY_OIDC_USERINFO_CACHE_TABLE" desc:"The database table the store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;PROXY_OIDC_USERINFO_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '10s' (10 seconds)."`
@@ -43,7 +43,7 @@ func DefaultConfig() *config.Config {
AccessTokenVerifyMethod: config.AccessTokenVerificationJWT,
UserinfoCache: &config.Cache{
Store: "memory",
Database: "proxy",
Database: "ocis",
Table: "userinfo",
TTL: time.Second * 10,
},
+1 -1
View File
@@ -86,7 +86,7 @@ type OCISDriver struct {
// Cache holds cache config
type Cache struct {
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_SYSTEM_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_SYSTEM_CACHE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_SYSTEM_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_SYSTEM_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '10s' (10 seconds)."`
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_SYSTEM_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512."`
@@ -52,7 +52,7 @@ func DefaultConfig() *config.Config {
},
Cache: config.Cache{
Store: "memory",
Database: "system",
Database: "ocis",
},
}
}
@@ -83,7 +83,7 @@ type Handler struct {
skipUpdatingExistingSharesMountpoints bool
additionalInfoTemplate *template.Template
userIdentifierCache *ttlcache.Cache
resourceInfoCache cache.StatCache
statCache cache.StatCache
deniable bool
resharing bool
@@ -130,7 +130,7 @@ func (h *Handler) Init(c *config.Config) {
h.deniable = c.EnableDenials
h.resharing = resharing(c)
h.resourceInfoCache = cache.GetStatCache(c.ResourceInfoCacheStore, c.ResourceInfoCacheNodes, c.ResourceInfoCacheDatabase, "stat", time.Duration(c.ResourceInfoCacheTTL)*time.Second, c.ResourceInfoCacheSize)
h.statCache = cache.GetStatCache(c.ResourceInfoCacheStore, c.ResourceInfoCacheNodes, c.ResourceInfoCacheDatabase, "stat", time.Duration(c.ResourceInfoCacheTTL)*time.Second, c.ResourceInfoCacheSize)
if c.CacheWarmupDriver != "" {
cwm, err := getCacheWarmupManager(c)
if err == nil {
@@ -153,8 +153,8 @@ func (h *Handler) startCacheWarmup(c sharecache.Warmup) {
return
}
for _, r := range infos {
key := h.resourceInfoCache.GetKey(r.Owner, &provider.Reference{ResourceId: r.Id}, []string{}, []string{})
_ = h.resourceInfoCache.PushToCache(key, r)
key := h.statCache.GetKey(r.Owner, &provider.Reference{ResourceId: r.Id}, []string{}, []string{})
_ = h.statCache.PushToCache(key, r)
}
}
@@ -769,7 +769,7 @@ func (h *Handler) updateShare(w http.ResponseWriter, r *http.Request, shareID st
}
if currentUser, ok := ctxpkg.ContextGetUser(ctx); ok {
h.resourceInfoCache.RemoveStat(currentUser.Id, shareR.Share.ResourceId)
h.statCache.RemoveStat(currentUser.Id, shareR.Share.ResourceId)
}
share, err := conversions.CS3Share2ShareData(ctx, uRes.Share)
@@ -1354,10 +1354,10 @@ func (h *Handler) getResourceInfo(ctx context.Context, client gateway.GatewayAPI
logger := appctx.GetLogger(ctx)
key := ""
if currentUser, ok := ctxpkg.ContextGetUser(ctx); ok {
key = h.resourceInfoCache.GetKey(currentUser.Id, ref, []string{}, []string{})
pinfo := &provider.ResourceInfo{}
if err := h.resourceInfoCache.PullFromCache(key, pinfo); err == nil {
return pinfo, &rpc.Status{Code: rpc.Code_CODE_OK}, nil
key = h.statCache.GetKey(currentUser.Id, ref, []string{}, []string{})
s := &provider.StatResponse{}
if err := h.statCache.PullFromCache(key, s); err == nil {
return s.Info, &rpc.Status{Code: rpc.Code_CODE_OK}, nil
}
}
@@ -1376,7 +1376,7 @@ func (h *Handler) getResourceInfo(ctx context.Context, client gateway.GatewayAPI
}
if key != "" {
_ = h.resourceInfoCache.PushToCache(key, *statRes.Info)
_ = h.statCache.PushToCache(key, statRes)
}
return statRes.Info, statRes.Status, nil
@@ -207,7 +207,7 @@ func (h *Handler) removeUserShare(w http.ResponseWriter, r *http.Request, shareI
return
}
if currentUser, ok := ctxpkg.ContextGetUser(ctx); ok {
h.resourceInfoCache.RemoveStat(currentUser.Id, getShareResp.Share.ResourceId)
h.statCache.RemoveStat(currentUser.Id, getShareResp.Share.ResourceId)
}
response.WriteOCSSuccess(w, r, data)
}
+107 -41
View File
@@ -1,15 +1,24 @@
// Package redis is a redis backed store implementation
package redis
import (
"context"
"fmt"
"time"
"github.com/go-redis/redis/v8"
log "go-micro.dev/v4/logger"
"go-micro.dev/v4/logger"
"go-micro.dev/v4/store"
"go-micro.dev/v4/util/cmd"
)
// DefaultDatabase is the namespace that the store
// will use if no namespace is provided.
var (
DefaultDatabase = "micro"
DefaultTable = "micro"
)
type rkv struct {
ctx context.Context
options store.Options
@@ -33,8 +42,9 @@ func (r *rkv) Close() error {
}
func (r *rkv) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
options := store.ReadOptions{}
options.Table = r.options.Table
options := store.ReadOptions{
Table: r.options.Table,
}
for _, o := range opts {
o(&options)
@@ -42,40 +52,57 @@ func (r *rkv) Read(key string, opts ...store.ReadOption) ([]*store.Record, error
var keys []string
rkey := fmt.Sprintf("%s%s", options.Table, key)
// Handle Prefix
// TODO suffix
if options.Prefix {
prefixKey := fmt.Sprintf("%s*", rkey)
fkeys, err := r.Client.Keys(r.ctx, prefixKey).Result()
if err != nil {
return nil, err
}
// TODO Limit Offset
var rkey string
keys = append(keys, fkeys...)
} else {
keys = []string{rkey}
switch {
case options.Prefix:
rkey = fmt.Sprintf("%s%s*", options.Table, key)
case options.Suffix:
rkey = fmt.Sprintf("%s*%s", options.Table, key)
default:
keys = []string{fmt.Sprintf("%s%s", options.Table, key)}
}
if len(keys) == 0 {
cursor := uint64(options.Offset)
count := int64(options.Limit)
for {
var err error
var ks []string
ks, cursor, err = r.Client.Scan(r.ctx, cursor, rkey, count).Result()
if err != nil {
return nil, err
}
keys = append(keys, ks...)
if cursor == 0 {
break
}
}
}
records := make([]*store.Record, 0, len(keys))
// read all keys, continue on error
var val []byte
var d time.Duration
var err error
for _, rkey = range keys {
val, err := r.Client.Get(r.ctx, rkey).Bytes()
if err != nil && err == redis.Nil {
return nil, store.ErrNotFound
} else if err != nil {
return nil, err
val, err = r.Client.Get(r.ctx, rkey).Bytes()
if err != nil || val == nil {
continue
}
if val == nil {
return nil, store.ErrNotFound
}
d, err := r.Client.TTL(r.ctx, rkey).Result()
d, err = r.Client.TTL(r.ctx, rkey).Result()
if err != nil {
return nil, err
continue
}
records = append(records, &store.Record{
@@ -85,47 +112,77 @@ func (r *rkv) Read(key string, opts ...store.ReadOption) ([]*store.Record, error
})
}
if len(keys) == 1 {
return records, err
}
// keys might have vanished since we scanned them, ignore errors
return records, nil
}
func (r *rkv) Delete(key string, opts ...store.DeleteOption) error {
options := store.DeleteOptions{}
options.Table = r.options.Table
options := store.DeleteOptions{
Table: r.options.Table,
}
for _, o := range opts {
o(&options)
}
rkey := fmt.Sprintf("%s%s", options.Table, key)
return r.Client.Del(r.ctx, rkey).Err()
}
func (r *rkv) Write(record *store.Record, opts ...store.WriteOption) error {
options := store.WriteOptions{}
options.Table = r.options.Table
options := store.WriteOptions{
Table: r.options.Table,
}
for _, o := range opts {
o(&options)
}
rkey := fmt.Sprintf("%s%s", options.Table, record.Key)
return r.Client.Set(r.ctx, rkey, record.Value, record.Expiry).Err()
}
func (r *rkv) List(opts ...store.ListOption) ([]string, error) {
options := store.ListOptions{}
options.Table = r.options.Table
options := store.ListOptions{
Table: r.options.Table,
}
for _, o := range opts {
o(&options)
}
keys, err := r.Client.Keys(r.ctx, "*").Result()
if err != nil {
return nil, err
key := fmt.Sprintf("%s%s*%s", options.Table, options.Prefix, options.Suffix)
cursor := uint64(options.Offset)
count := int64(options.Limit)
var allKeys []string
var keys []string
var err error
for {
keys, cursor, err = r.Client.Scan(r.ctx, cursor, key, count).Result()
if err != nil {
return nil, err
}
allKeys = append(allKeys, keys...)
if cursor == 0 {
break
}
}
return keys, nil
return allKeys, nil
}
func (r *rkv) Options() store.Options {
@@ -136,8 +193,14 @@ func (r *rkv) String() string {
return "redis"
}
// NewStore returns a redis store.
func NewStore(opts ...store.Option) store.Store {
var options store.Options
options := store.Options{
Database: DefaultDatabase,
Table: DefaultTable,
Logger: logger.DefaultLogger,
}
for _, o := range opts {
o(&options)
}
@@ -148,7 +211,7 @@ func NewStore(opts ...store.Option) store.Store {
}
if err := s.configure(); err != nil {
log.Fatal(err)
s.options.Logger.Log(logger.ErrorLevel, "Error configuring store ", err)
}
return s
@@ -156,8 +219,11 @@ func NewStore(opts ...store.Option) store.Store {
func (r *rkv) configure() error {
if r.Client != nil {
r.Client.Close()
if err := r.Client.Close(); err != nil {
return err
}
}
r.Client = newUniversalClient(r.options)
return nil
+2 -2
View File
@@ -349,7 +349,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1
github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1
github.com/cs3org/go-cs3apis/cs3/tx/v1beta1
github.com/cs3org/go-cs3apis/cs3/types/v1beta1
# github.com/cs3org/reva/v2 v2.12.1-0.20230424091007-8d8b567179b1
# github.com/cs3org/reva/v2 v2.12.1-0.20230425110812-8a3352fb1960
## explicit; go 1.19
github.com/cs3org/reva/v2/cmd/revad/internal/grace
github.com/cs3org/reva/v2/cmd/revad/runtime
@@ -909,7 +909,7 @@ github.com/go-micro/plugins/v4/server/http
# github.com/go-micro/plugins/v4/store/nats-js v1.1.0
## explicit; go 1.17
github.com/go-micro/plugins/v4/store/nats-js
# github.com/go-micro/plugins/v4/store/redis v1.2.0
# github.com/go-micro/plugins/v4/store/redis v1.2.1-0.20230405210006-efd9191305c5
## explicit; go 1.17
github.com/go-micro/plugins/v4/store/redis
# github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0