From 003310a2b054dfd6112b1eb6c89823599ef9fb70 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 23 Nov 2023 12:33:42 +0100 Subject: [PATCH 1/5] use nats-js cache Signed-off-by: jkoberg --- .drone.star | 3 +++ changelog/unreleased/fix-natsjs-cache.md | 5 +++++ ocis/pkg/runtime/service/service.go | 6 ++++-- services/gateway/pkg/config/config.go | 2 +- .../storage-users/pkg/config/defaults/defaultconfig.go | 5 +++++ tests/ociswrapper/ocis/ocis.go | 2 +- .../utils/decomposedfs/metadata/messagepack_backend.go | 9 ++------- 7 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 changelog/unreleased/fix-natsjs-cache.md diff --git a/.drone.star b/.drone.star index faa2eb964..070140121 100644 --- a/.drone.star +++ b/.drone.star @@ -1919,6 +1919,9 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "OCIS_EVENTS_ENABLE_TLS": False, "MICRO_REGISTRY": "natsjs", "MICRO_REGISTRY_ADDRESS": "127.0.0.1:9233", + "OCIS_CACHE_STORE": "nats-js-kv", + "OCIS_CACHE_STORE_NODES": "127.0.0.1:9233", + "OCIS_CACHE_DATABASE": "cache", } if deploy_type == "": diff --git a/changelog/unreleased/fix-natsjs-cache.md b/changelog/unreleased/fix-natsjs-cache.md new file mode 100644 index 000000000..002867d02 --- /dev/null +++ b/changelog/unreleased/fix-natsjs-cache.md @@ -0,0 +1,5 @@ +Bugfix: Fix natsjs cache + +The nats-js cache was not working. It paniced and wrote a lot of error logs. Both is fixed now. + +https://github.com/owncloud/ocis/pull/7790 diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index 5c74c6e0f..c1ef41747 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -64,6 +64,8 @@ import ( var ( // runset keeps track of which services to start supervised. runset map[string]struct{} + // time to wait after starting the preliminary services + _preliminaryDelay = 6 * time.Second // time to wait between starting service groups (preliminary, main, delayed) _startDelay = 2 * time.Second ) @@ -413,8 +415,8 @@ func Start(o ...Option) error { // trap will block on halt channel for interruptions. go trap(s, halt) - // grace period for supervisor to get up - time.Sleep(_startDelay) + // grace period for preliminary services to get up + time.Sleep(_preliminaryDelay) // schedule services that we are sure don't have interdependencies. scheduleServiceTokens(s, s.ServicesRegistry) diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index 73c6a01ee..f30bb8b36 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -85,7 +85,7 @@ type StorageRegistry struct { // Cache holds cache config type Cache struct { - StatCacheStore string `yaml:"stat_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + StatCacheStore string // TODO: enable configuring this again // `yaml:"stat_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 1bb34eab5..1268b20cf 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -103,6 +103,11 @@ func DefaultConfig() *config.Config { Database: "ocis", TTL: 24 * 60 * time.Second, }, + IDCache: config.IDCache{ + Store: "nats-js-kv", + Database: "ocis", + TTL: 24 * 60 * time.Second, + }, Tasks: config.Tasks{ PurgeTrashBin: config.PurgeTrashBin{ ProjectDeleteBefore: 30 * 24 * time.Hour, diff --git a/tests/ociswrapper/ocis/ocis.go b/tests/ociswrapper/ocis/ocis.go index c9762b6fd..815b8c1ab 100644 --- a/tests/ociswrapper/ocis/ocis.go +++ b/tests/ociswrapper/ocis/ocis.go @@ -128,7 +128,7 @@ func WaitForConnection() bool { Transport: transport, } - req, _ := http.NewRequest("GET", config.Get("url")+"/graph/v1.0/users/"+config.Get("adminUsername"), nil) + req, _ := http.NewRequest("GET", config.Get("url")+"/graph/v1.0/me/drives", nil) req.SetBasicAuth(config.Get("adminUsername"), config.Get("adminPassword")) timeout := time.After(timeoutValue) diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go index 92caf0cf5..d0c491ec7 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go @@ -275,9 +275,7 @@ func (MessagePackBackend) IsMetaFile(path string) bool { // Purge purges the data of a given path func (b MessagePackBackend) Purge(path string) error { - if err := b.metaCache.RemoveMetadata(b.cacheKey(path)); err != nil { - return err - } + _ = b.metaCache.RemoveMetadata(b.cacheKey(path)) return os.Remove(b.MetadataPath(path)) } @@ -291,10 +289,7 @@ func (b MessagePackBackend) Rename(oldPath, newPath string) error { return err } } - err = b.metaCache.RemoveMetadata(b.cacheKey(oldPath)) - if err != nil { - return err - } + _ = b.metaCache.RemoveMetadata(b.cacheKey(oldPath)) return os.Rename(b.MetadataPath(oldPath), b.MetadataPath(newPath)) } From bbc4f83f7d971cc3cae52e88df46b365dcdfa41a Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 23 Nov 2023 12:34:01 +0100 Subject: [PATCH 2/5] bump dependencies Signed-off-by: jkoberg --- .drone.star | 1 - changelog/unreleased/bump-reva.md | 5 ++++ go.mod | 4 +-- go.sum | 4 +-- .../pkg/config/defaults/defaultconfig.go | 4 +-- services/gateway/pkg/config/config.go | 2 +- .../pkg/config/defaults/defaultconfig.go | 8 ++--- .../pkg/config/defaults/defaultconfig.go | 4 +-- .../pkg/config/defaults/defaultconfig.go | 4 +-- .../pkg/config/defaults/defaultconfig.go | 4 +-- .../pkg/config/defaults/defaultconfig.go | 4 +-- .../pkg/config/defaults/defaultconfig.go | 6 ++-- .../metadata/messagepack_backend.go | 9 ++++-- .../go-micro/plugins/v4/store/nats-js/nats.go | 30 +++++++++---------- vendor/modules.txt | 5 ++-- 15 files changed, 49 insertions(+), 45 deletions(-) create mode 100644 changelog/unreleased/bump-reva.md diff --git a/.drone.star b/.drone.star index 070140121..19bd4934f 100644 --- a/.drone.star +++ b/.drone.star @@ -1921,7 +1921,6 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "MICRO_REGISTRY_ADDRESS": "127.0.0.1:9233", "OCIS_CACHE_STORE": "nats-js-kv", "OCIS_CACHE_STORE_NODES": "127.0.0.1:9233", - "OCIS_CACHE_DATABASE": "cache", } if deploy_type == "": diff --git a/changelog/unreleased/bump-reva.md b/changelog/unreleased/bump-reva.md new file mode 100644 index 000000000..5e3f954f8 --- /dev/null +++ b/changelog/unreleased/bump-reva.md @@ -0,0 +1,5 @@ +Enhancement: Bump reva + +Bumps reva version + +https://github.com/owncloud/ocis/pull/7793 diff --git a/go.mod b/go.mod index c3517e9bd..f3067a37c 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/go-micro/plugins/v4/registry/nats v1.2.2-0.20230723205323-1ada01245674 github.com/go-micro/plugins/v4/server/grpc v1.2.0 github.com/go-micro/plugins/v4/server/http v1.2.2 - github.com/go-micro/plugins/v4/store/nats-js v1.2.1-0.20230807070816-bc05fb076ce7 + github.com/go-micro/plugins/v4/store/nats-js v1.2.1-0.20231129143103-d72facc652f0 github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0 github.com/go-micro/plugins/v4/wrapper/monitoring/prometheus v1.2.0 github.com/go-micro/plugins/v4/wrapper/trace/opentelemetry v1.2.0 @@ -347,6 +347,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace github.com/go-micro/plugins/v4/store/nats-js => github.com/kobergj/plugins/v4/store/nats-js v1.2.1-0.20231020092801-9463c820c19a - replace github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 diff --git a/go.sum b/go.sum index 3e4b489a2..1c14b0138 100644 --- a/go.sum +++ b/go.sum @@ -1213,6 +1213,8 @@ github.com/go-micro/plugins/v4/server/grpc v1.2.0 h1:lXfM+/0oE/u1g0hVBYsvbP4lYOY github.com/go-micro/plugins/v4/server/grpc v1.2.0/go.mod h1:+Ah9Pf/vMSXxBM3fup/hc3N+zN2as3nIpcRaR4sBjnY= github.com/go-micro/plugins/v4/server/http v1.2.2 h1:UK2/09AU0zV3wHELuR72TZzVU2vTUhbx9qrRGrQSIWg= github.com/go-micro/plugins/v4/server/http v1.2.2/go.mod h1:YuAjaSPxcn3LI8j2FUsqx0Rxunrj4YwDV41Ax76rLl0= +github.com/go-micro/plugins/v4/store/nats-js v1.2.1-0.20231129143103-d72facc652f0 h1:Qa1EBQ9UyCGecFAJQovl/MHGnvbcvDaM3qUoAG5Lnvk= +github.com/go-micro/plugins/v4/store/nats-js v1.2.1-0.20231129143103-d72facc652f0/go.mod h1:aCRl8JQmqIaonOl88nFPY/BOQnHPVHY9ngStzLkXnYk= github.com/go-micro/plugins/v4/store/redis v1.2.1-0.20230510195111-07cd57e1bc9d h1:HQoDDVyMfdkrgXNo03ZY4vzhoOXMDZVZ4SnpBDVID6E= github.com/go-micro/plugins/v4/store/redis v1.2.1-0.20230510195111-07cd57e1bc9d/go.mod h1:MbCG0YiyPqETTtm7uHFmxQNCaW1o9hBoYtFwhbVjLUg= github.com/go-micro/plugins/v4/transport/grpc v1.1.0 h1:mXfDYfFQLnVDzjGY3o84oe4prfux9h8txsnA19dKsj8= @@ -1593,8 +1595,6 @@ github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/kobergj/plugins/v4/store/nats-js v1.2.1-0.20231020092801-9463c820c19a h1:W+itvdTLFGLuFh+E5IzW08n2BS02cHK91qnMo7SUxbA= -github.com/kobergj/plugins/v4/store/nats-js v1.2.1-0.20231020092801-9463c820c19a/go.mod h1:wt51O2yNmgF/F7E00IYIH0awseRGqtnmjZGn6RjbZSk= github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 h1:Czv6AW9Suj6npWd5BLZjobdD78c2RdzBeKBgkq3jYZk= github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348/go.mod h1:Goi4eJ9SrKkxE6NsAVqBVNxfQFbwb7UbyII6743ldgM= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= diff --git a/services/frontend/pkg/config/defaults/defaultconfig.go b/services/frontend/pkg/config/defaults/defaultconfig.go index b589aa2f9..a99acce7b 100644 --- a/services/frontend/pkg/config/defaults/defaultconfig.go +++ b/services/frontend/pkg/config/defaults/defaultconfig.go @@ -111,8 +111,8 @@ func DefaultConfig() *config.Config { SharePrefix: "/Shares", HomeNamespace: "/users/{{.Id.OpaqueId}}", AdditionalInfoAttribute: "{{.Mail}}", - StatCacheType: "noop", - StatCacheDatabase: "ocis", + StatCacheType: "nats-js-kv", + StatCacheDatabase: "cache-stat", StatCacheTTL: 300 * time.Second, ListOCMShares: true, PublicShareMustHavePassword: true, diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index f30bb8b36..81e596e43 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -90,7 +90,7 @@ type Cache struct { StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` - ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + ProviderCacheStore string // `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` diff --git a/services/gateway/pkg/config/defaults/defaultconfig.go b/services/gateway/pkg/config/defaults/defaultconfig.go index 736638cb2..4700efb58 100644 --- a/services/gateway/pkg/config/defaults/defaultconfig.go +++ b/services/gateway/pkg/config/defaults/defaultconfig.go @@ -39,14 +39,14 @@ func DefaultConfig() *config.Config { DisableHomeCreationOnLogin: true, TransferExpires: 24 * 60 * 60, Cache: config.Cache{ - StatCacheStore: "noop", + StatCacheStore: "noop", // NOTE: stat cache not working StatCacheDatabase: "ocis", StatCacheTTL: 300 * time.Second, ProviderCacheStore: "noop", - ProviderCacheDatabase: "ocis", + ProviderCacheDatabase: "cache-providers", ProviderCacheTTL: 300 * time.Second, - CreateHomeCacheStore: "noop", - CreateHomeCacheDatabase: "ocis", + CreateHomeCacheStore: "nats-js-kv", + CreateHomeCacheDatabase: "cache-createhome", CreateHomeCacheTTL: 300 * time.Second, }, diff --git a/services/graph/pkg/config/defaults/defaultconfig.go b/services/graph/pkg/config/defaults/defaultconfig.go index 3062e5ed9..10e823059 100644 --- a/services/graph/pkg/config/defaults/defaultconfig.go +++ b/services/graph/pkg/config/defaults/defaultconfig.go @@ -96,8 +96,8 @@ func DefaultConfig() *config.Config { }, }, Cache: &config.Cache{ - Store: "memory", - Database: "ocis", + Store: "nats-js-kv", + Database: "cache-roles", Table: "roles", TTL: time.Hour * 336, }, diff --git a/services/proxy/pkg/config/defaults/defaultconfig.go b/services/proxy/pkg/config/defaults/defaultconfig.go index f7e048730..b2b24baef 100644 --- a/services/proxy/pkg/config/defaults/defaultconfig.go +++ b/services/proxy/pkg/config/defaults/defaultconfig.go @@ -43,8 +43,8 @@ func DefaultConfig() *config.Config { AccessTokenVerifyMethod: config.AccessTokenVerificationJWT, SkipUserInfo: false, UserinfoCache: &config.Cache{ - Store: "memory", - Database: "ocis", + Store: "nats-js-kv", + Database: "cache-userinfo", Table: "userinfo", TTL: time.Second * 10, }, diff --git a/services/settings/pkg/config/defaults/defaultconfig.go b/services/settings/pkg/config/defaults/defaultconfig.go index 8bd1126fe..1e33bb0ec 100644 --- a/services/settings/pkg/config/defaults/defaultconfig.go +++ b/services/settings/pkg/config/defaults/defaultconfig.go @@ -57,8 +57,8 @@ func DefaultConfig() *config.Config { StorageAddress: "com.owncloud.api.storage-system", SystemUserIDP: "internal", Cache: &config.Cache{ - Store: "memory", - Database: "ocis", + Store: "nats-js-kv", + Database: "settings-cache", FileTable: "settings_files", DirectoryTable: "settings_dirs", TTL: time.Minute * 10, diff --git a/services/storage-system/pkg/config/defaults/defaultconfig.go b/services/storage-system/pkg/config/defaults/defaultconfig.go index 63fa5eba6..a5645dc8c 100644 --- a/services/storage-system/pkg/config/defaults/defaultconfig.go +++ b/services/storage-system/pkg/config/defaults/defaultconfig.go @@ -52,8 +52,8 @@ func DefaultConfig() *config.Config { }, }, FileMetadataCache: config.Cache{ - Store: "memory", - Database: "ocis", + Store: "nats-js-kv", + Database: "cache-filemetadata", TTL: 24 * 60 * time.Second, }, } diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 1268b20cf..08b605d5f 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -99,13 +99,13 @@ func DefaultConfig() *config.Config { TTL: 300 * time.Second, }, FilemetadataCache: config.FilemetadataCache{ - Store: "memory", - Database: "ocis", + Store: "nats-js-kv", + Database: "cache-filemetadata", TTL: 24 * 60 * time.Second, }, IDCache: config.IDCache{ Store: "nats-js-kv", - Database: "ocis", + Database: "ids-storage-users", TTL: 24 * 60 * time.Second, }, Tasks: config.Tasks{ diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go index d0c491ec7..92caf0cf5 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go @@ -275,7 +275,9 @@ func (MessagePackBackend) IsMetaFile(path string) bool { // Purge purges the data of a given path func (b MessagePackBackend) Purge(path string) error { - _ = b.metaCache.RemoveMetadata(b.cacheKey(path)) + if err := b.metaCache.RemoveMetadata(b.cacheKey(path)); err != nil { + return err + } return os.Remove(b.MetadataPath(path)) } @@ -289,7 +291,10 @@ func (b MessagePackBackend) Rename(oldPath, newPath string) error { return err } } - _ = b.metaCache.RemoveMetadata(b.cacheKey(oldPath)) + err = b.metaCache.RemoveMetadata(b.cacheKey(oldPath)) + if err != nil { + return err + } return os.Rename(b.MetadataPath(oldPath), b.MetadataPath(newPath)) } diff --git a/vendor/github.com/go-micro/plugins/v4/store/nats-js/nats.go b/vendor/github.com/go-micro/plugins/v4/store/nats-js/nats.go index c0a4f4004..6fc2cf336 100644 --- a/vendor/github.com/go-micro/plugins/v4/store/nats-js/nats.go +++ b/vendor/github.com/go-micro/plugins/v4/store/nats-js/nats.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/cornelk/hashmap" "github.com/nats-io/nats.go" "github.com/pkg/errors" "go-micro.dev/v4/store" @@ -34,7 +35,7 @@ type natsStore struct { conn *nats.Conn js nats.JetStreamContext - buckets *sync.Map + buckets *hashmap.Map[string, nats.ObjectStore] } func init() { @@ -55,7 +56,7 @@ func NewStore(opts ...store.Option) store.Store { opts: options, jsopts: []nats.JSOpt{}, objStoreConfigs: []*nats.ObjectStoreConfig{}, - buckets: &sync.Map{}, + buckets: hashmap.New[string, nats.ObjectStore](), storageType: nats.FileStorage, } @@ -103,7 +104,7 @@ func (n *natsStore) Init(opts ...store.Option) error { if err != nil { return errors.Wrapf(err, "Failed to create bucket (%s)", cfg.Bucket) } - n.buckets.Store(cfg.Bucket, store) + n.buckets.Set(cfg.Bucket, store) } return nil @@ -178,11 +179,10 @@ func (n *natsStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, opt.Table = n.opts.Table } - b, ok := n.buckets.Load(opt.Database) + bucket, ok := n.buckets.Get(opt.Database) if !ok { return nil, ErrBucketNotFound } - bucket := b.(nats.ObjectStore) var keys []string @@ -292,8 +292,7 @@ func (n *natsStore) Write(r *store.Record, opts ...store.WriteOption) error { opt.Table = n.opts.Table } - s, ok := n.buckets.Load(opt.Database) - store, _ := s.(nats.ObjectStore) + store, ok := n.buckets.Get(opt.Database) // Create new bucket if not exists if !ok { var err error @@ -345,19 +344,18 @@ func (n *natsStore) Delete(key string, opts ...store.DeleteOption) error { } if opt.Table == "DELETE_BUCKET" { - n.buckets.Delete(key) + n.buckets.Del(key) if err := n.js.DeleteObjectStore(key); err != nil { return errors.Wrap(err, "Failed to delete bucket") } return nil } - s, ok := n.buckets.Load(opt.Database) + store, ok := n.buckets.Get(opt.Database) if !ok { return ErrBucketNotFound } - store := s.(nats.ObjectStore) if err := store.Delete(getKey(key, opt.Table)); err != nil { return errors.Wrap(err, "Failed to delete data") } @@ -383,11 +381,10 @@ func (n *natsStore) List(opts ...store.ListOption) ([]string, error) { opt.Table = n.opts.Table } - s, ok := n.buckets.Load(opt.Database) + store, ok := n.buckets.Get(opt.Database) if !ok { return nil, ErrBucketNotFound } - store := s.(nats.ObjectStore) objects, err := store.List() if err != nil { @@ -405,13 +402,14 @@ func (n *natsStore) List(opts ...store.ListOption) ([]string, error) { if !strings.HasSuffix(key, opt.Suffix) { continue } - keys = append(keys, key) + + keys = append(keys, strings.TrimPrefix(key, getKey(opt.Prefix, opt.Table))) } - if opt.Limit > 0 { + if opt.Limit > 0 && int(opt.Offset+opt.Limit) < len(keys) { return keys[opt.Offset : opt.Offset+opt.Limit], nil } - if opt.Offset > 0 { + if opt.Offset > 0 && int(opt.Offset) < len(keys) { return keys[opt.Offset:], nil } return keys, nil @@ -448,7 +446,7 @@ func (n *natsStore) createNewBucket(name string) (nats.ObjectStore, error) { if err != nil { return nil, errors.Wrapf(err, "Failed to create new bucket (%s)", name) } - n.buckets.Store(name, store) + n.buckets.Set(name, store) return store, err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 2868e0177..5e130cae5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -937,8 +937,8 @@ github.com/go-micro/plugins/v4/server/grpc # github.com/go-micro/plugins/v4/server/http v1.2.2 ## explicit; go 1.17 github.com/go-micro/plugins/v4/server/http -# github.com/go-micro/plugins/v4/store/nats-js v1.2.1-0.20230807070816-bc05fb076ce7 => github.com/kobergj/plugins/v4/store/nats-js v1.2.1-0.20231020092801-9463c820c19a -## explicit; go 1.17 +# github.com/go-micro/plugins/v4/store/nats-js v1.2.1-0.20231129143103-d72facc652f0 +## explicit; go 1.21 github.com/go-micro/plugins/v4/store/nats-js # github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-00010101000000-000000000000 => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 ## explicit; go 1.21 @@ -2297,5 +2297,4 @@ stash.kopano.io/kgol/oidc-go # stash.kopano.io/kgol/rndm v1.1.2 ## explicit; go 1.13 stash.kopano.io/kgol/rndm -# github.com/go-micro/plugins/v4/store/nats-js => github.com/kobergj/plugins/v4/store/nats-js v1.2.1-0.20231020092801-9463c820c19a # github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 From ad87ac955a17f465d8f6335bf8138fb0f18f0b9f Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 7 Dec 2023 11:59:03 +0100 Subject: [PATCH 3/5] configure services individually Signed-off-by: jkoberg --- .drone.star | 2 -- services/frontend/pkg/config/defaults/defaultconfig.go | 1 + services/gateway/pkg/config/config.go | 4 ++-- services/gateway/pkg/config/defaults/defaultconfig.go | 3 +++ services/graph/pkg/config/defaults/defaultconfig.go | 2 +- services/proxy/pkg/config/defaults/defaultconfig.go | 2 +- services/settings/pkg/config/defaults/defaultconfig.go | 1 + services/storage-system/pkg/config/defaults/defaultconfig.go | 1 + services/storage-users/pkg/config/defaults/defaultconfig.go | 5 ++++- 9 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.drone.star b/.drone.star index 19bd4934f..faa2eb964 100644 --- a/.drone.star +++ b/.drone.star @@ -1919,8 +1919,6 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "OCIS_EVENTS_ENABLE_TLS": False, "MICRO_REGISTRY": "natsjs", "MICRO_REGISTRY_ADDRESS": "127.0.0.1:9233", - "OCIS_CACHE_STORE": "nats-js-kv", - "OCIS_CACHE_STORE_NODES": "127.0.0.1:9233", } if deploy_type == "": diff --git a/services/frontend/pkg/config/defaults/defaultconfig.go b/services/frontend/pkg/config/defaults/defaultconfig.go index a99acce7b..d4cb02c57 100644 --- a/services/frontend/pkg/config/defaults/defaultconfig.go +++ b/services/frontend/pkg/config/defaults/defaultconfig.go @@ -112,6 +112,7 @@ func DefaultConfig() *config.Config { HomeNamespace: "/users/{{.Id.OpaqueId}}", AdditionalInfoAttribute: "{{.Mail}}", StatCacheType: "nats-js-kv", + StatCacheNodes: []string{"127.0.0.1:9233"}, StatCacheDatabase: "cache-stat", StatCacheTTL: 300 * time.Second, ListOCMShares: true, diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index 81e596e43..b9393ca02 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -85,12 +85,12 @@ type StorageRegistry struct { // Cache holds cache config type Cache struct { - StatCacheStore string // TODO: enable configuring this again // `yaml:"stat_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + StatCacheStore string // NOTE: The stat cache is not working atm. Hence we block configuring it StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` - ProviderCacheStore string // `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` diff --git a/services/gateway/pkg/config/defaults/defaultconfig.go b/services/gateway/pkg/config/defaults/defaultconfig.go index 4700efb58..ac55a8946 100644 --- a/services/gateway/pkg/config/defaults/defaultconfig.go +++ b/services/gateway/pkg/config/defaults/defaultconfig.go @@ -41,11 +41,14 @@ func DefaultConfig() *config.Config { Cache: config.Cache{ StatCacheStore: "noop", // NOTE: stat cache not working StatCacheDatabase: "ocis", + StatCacheNodes: []string{"127.0.0.1:9233"}, StatCacheTTL: 300 * time.Second, ProviderCacheStore: "noop", + ProviderCacheNodes: []string{"127.0.0.1:9233"}, ProviderCacheDatabase: "cache-providers", ProviderCacheTTL: 300 * time.Second, CreateHomeCacheStore: "nats-js-kv", + CreateHomeCacheNodes: []string{"127.0.0.1:9233"}, CreateHomeCacheDatabase: "cache-createhome", CreateHomeCacheTTL: 300 * time.Second, }, diff --git a/services/graph/pkg/config/defaults/defaultconfig.go b/services/graph/pkg/config/defaults/defaultconfig.go index 10e823059..77acf8084 100644 --- a/services/graph/pkg/config/defaults/defaultconfig.go +++ b/services/graph/pkg/config/defaults/defaultconfig.go @@ -97,8 +97,8 @@ func DefaultConfig() *config.Config { }, Cache: &config.Cache{ Store: "nats-js-kv", + Nodes: []string{"127.0.0.1:9233"}, Database: "cache-roles", - Table: "roles", TTL: time.Hour * 336, }, Events: config.Events{ diff --git a/services/proxy/pkg/config/defaults/defaultconfig.go b/services/proxy/pkg/config/defaults/defaultconfig.go index b2b24baef..bac52f0c2 100644 --- a/services/proxy/pkg/config/defaults/defaultconfig.go +++ b/services/proxy/pkg/config/defaults/defaultconfig.go @@ -44,8 +44,8 @@ func DefaultConfig() *config.Config { SkipUserInfo: false, UserinfoCache: &config.Cache{ Store: "nats-js-kv", + Nodes: []string{"127.0.0.1:9233"}, Database: "cache-userinfo", - Table: "userinfo", TTL: time.Second * 10, }, JWKS: config.JWKS{ diff --git a/services/settings/pkg/config/defaults/defaultconfig.go b/services/settings/pkg/config/defaults/defaultconfig.go index 1e33bb0ec..030ce4344 100644 --- a/services/settings/pkg/config/defaults/defaultconfig.go +++ b/services/settings/pkg/config/defaults/defaultconfig.go @@ -58,6 +58,7 @@ func DefaultConfig() *config.Config { SystemUserIDP: "internal", Cache: &config.Cache{ Store: "nats-js-kv", + Nodes: []string{"127.0.0.1:9233"}, Database: "settings-cache", FileTable: "settings_files", DirectoryTable: "settings_dirs", diff --git a/services/storage-system/pkg/config/defaults/defaultconfig.go b/services/storage-system/pkg/config/defaults/defaultconfig.go index a5645dc8c..df911bd45 100644 --- a/services/storage-system/pkg/config/defaults/defaultconfig.go +++ b/services/storage-system/pkg/config/defaults/defaultconfig.go @@ -53,6 +53,7 @@ func DefaultConfig() *config.Config { }, FileMetadataCache: config.Cache{ Store: "nats-js-kv", + Nodes: []string{"127.0.0.1:9233"}, Database: "cache-filemetadata", TTL: 24 * 60 * time.Second, }, diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 08b605d5f..6644431e6 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -94,17 +94,20 @@ func DefaultConfig() *config.Config { EnableTLS: false, }, StatCache: config.StatCache{ - Store: "noop", + Store: "nats-js-kv", + Nodes: []string{"127.0.0.1:9233"}, Database: "ocis", TTL: 300 * time.Second, }, FilemetadataCache: config.FilemetadataCache{ Store: "nats-js-kv", + Nodes: []string{"127.0.0.1:9233"}, Database: "cache-filemetadata", TTL: 24 * 60 * time.Second, }, IDCache: config.IDCache{ Store: "nats-js-kv", + Nodes: []string{"127.0.0.1:9233"}, Database: "ids-storage-users", TTL: 24 * 60 * time.Second, }, From d62cb9d34b61b30a0b34bcb1b6049fc2968dfb24 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 13 Dec 2023 12:47:59 +0100 Subject: [PATCH 4/5] use inmemory cache by default Signed-off-by: jkoberg --- services/frontend/pkg/config/defaults/defaultconfig.go | 2 +- services/gateway/pkg/config/defaults/defaultconfig.go | 2 +- services/graph/pkg/config/defaults/defaultconfig.go | 2 +- services/proxy/pkg/config/defaults/defaultconfig.go | 2 +- services/settings/pkg/config/defaults/defaultconfig.go | 2 +- .../storage-system/pkg/config/defaults/defaultconfig.go | 2 +- services/storage-users/pkg/config/defaults/defaultconfig.go | 6 +++--- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/services/frontend/pkg/config/defaults/defaultconfig.go b/services/frontend/pkg/config/defaults/defaultconfig.go index d4cb02c57..344b24013 100644 --- a/services/frontend/pkg/config/defaults/defaultconfig.go +++ b/services/frontend/pkg/config/defaults/defaultconfig.go @@ -111,7 +111,7 @@ func DefaultConfig() *config.Config { SharePrefix: "/Shares", HomeNamespace: "/users/{{.Id.OpaqueId}}", AdditionalInfoAttribute: "{{.Mail}}", - StatCacheType: "nats-js-kv", + StatCacheType: "memory", StatCacheNodes: []string{"127.0.0.1:9233"}, StatCacheDatabase: "cache-stat", StatCacheTTL: 300 * time.Second, diff --git a/services/gateway/pkg/config/defaults/defaultconfig.go b/services/gateway/pkg/config/defaults/defaultconfig.go index ac55a8946..f1ae85004 100644 --- a/services/gateway/pkg/config/defaults/defaultconfig.go +++ b/services/gateway/pkg/config/defaults/defaultconfig.go @@ -47,7 +47,7 @@ func DefaultConfig() *config.Config { ProviderCacheNodes: []string{"127.0.0.1:9233"}, ProviderCacheDatabase: "cache-providers", ProviderCacheTTL: 300 * time.Second, - CreateHomeCacheStore: "nats-js-kv", + CreateHomeCacheStore: "memory", CreateHomeCacheNodes: []string{"127.0.0.1:9233"}, CreateHomeCacheDatabase: "cache-createhome", CreateHomeCacheTTL: 300 * time.Second, diff --git a/services/graph/pkg/config/defaults/defaultconfig.go b/services/graph/pkg/config/defaults/defaultconfig.go index 77acf8084..cb0ce6d0f 100644 --- a/services/graph/pkg/config/defaults/defaultconfig.go +++ b/services/graph/pkg/config/defaults/defaultconfig.go @@ -96,7 +96,7 @@ func DefaultConfig() *config.Config { }, }, Cache: &config.Cache{ - Store: "nats-js-kv", + Store: "memory", Nodes: []string{"127.0.0.1:9233"}, Database: "cache-roles", TTL: time.Hour * 336, diff --git a/services/proxy/pkg/config/defaults/defaultconfig.go b/services/proxy/pkg/config/defaults/defaultconfig.go index bac52f0c2..9c07ecb88 100644 --- a/services/proxy/pkg/config/defaults/defaultconfig.go +++ b/services/proxy/pkg/config/defaults/defaultconfig.go @@ -43,7 +43,7 @@ func DefaultConfig() *config.Config { AccessTokenVerifyMethod: config.AccessTokenVerificationJWT, SkipUserInfo: false, UserinfoCache: &config.Cache{ - Store: "nats-js-kv", + Store: "memory", Nodes: []string{"127.0.0.1:9233"}, Database: "cache-userinfo", TTL: time.Second * 10, diff --git a/services/settings/pkg/config/defaults/defaultconfig.go b/services/settings/pkg/config/defaults/defaultconfig.go index 030ce4344..47df608f7 100644 --- a/services/settings/pkg/config/defaults/defaultconfig.go +++ b/services/settings/pkg/config/defaults/defaultconfig.go @@ -57,7 +57,7 @@ func DefaultConfig() *config.Config { StorageAddress: "com.owncloud.api.storage-system", SystemUserIDP: "internal", Cache: &config.Cache{ - Store: "nats-js-kv", + Store: "memory", Nodes: []string{"127.0.0.1:9233"}, Database: "settings-cache", FileTable: "settings_files", diff --git a/services/storage-system/pkg/config/defaults/defaultconfig.go b/services/storage-system/pkg/config/defaults/defaultconfig.go index df911bd45..9840025a3 100644 --- a/services/storage-system/pkg/config/defaults/defaultconfig.go +++ b/services/storage-system/pkg/config/defaults/defaultconfig.go @@ -52,7 +52,7 @@ func DefaultConfig() *config.Config { }, }, FileMetadataCache: config.Cache{ - Store: "nats-js-kv", + Store: "memory", Nodes: []string{"127.0.0.1:9233"}, Database: "cache-filemetadata", TTL: 24 * 60 * time.Second, diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 6644431e6..b05f14cd2 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -94,19 +94,19 @@ func DefaultConfig() *config.Config { EnableTLS: false, }, StatCache: config.StatCache{ - Store: "nats-js-kv", + Store: "memory", Nodes: []string{"127.0.0.1:9233"}, Database: "ocis", TTL: 300 * time.Second, }, FilemetadataCache: config.FilemetadataCache{ - Store: "nats-js-kv", + Store: "memory", Nodes: []string{"127.0.0.1:9233"}, Database: "cache-filemetadata", TTL: 24 * 60 * time.Second, }, IDCache: config.IDCache{ - Store: "nats-js-kv", + Store: "memory", Nodes: []string{"127.0.0.1:9233"}, Database: "ids-storage-users", TTL: 24 * 60 * time.Second, From dbc50a655b75e982dd1279c7888bdc339ee74529 Mon Sep 17 00:00:00 2001 From: kobergj Date: Thu, 14 Dec 2023 10:52:57 +0100 Subject: [PATCH 5/5] use different dbs for filemetadata caches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörn Friedrich Dreyer --- services/storage-system/pkg/config/defaults/defaultconfig.go | 2 +- services/storage-users/pkg/config/defaults/defaultconfig.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage-system/pkg/config/defaults/defaultconfig.go b/services/storage-system/pkg/config/defaults/defaultconfig.go index 9840025a3..40f78410e 100644 --- a/services/storage-system/pkg/config/defaults/defaultconfig.go +++ b/services/storage-system/pkg/config/defaults/defaultconfig.go @@ -54,7 +54,7 @@ func DefaultConfig() *config.Config { FileMetadataCache: config.Cache{ Store: "memory", Nodes: []string{"127.0.0.1:9233"}, - Database: "cache-filemetadata", + Database: "storage-system", TTL: 24 * 60 * time.Second, }, } diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index b05f14cd2..993f4fd9a 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -102,7 +102,7 @@ func DefaultConfig() *config.Config { FilemetadataCache: config.FilemetadataCache{ Store: "memory", Nodes: []string{"127.0.0.1:9233"}, - Database: "cache-filemetadata", + Database: "storage-users", TTL: 24 * 60 * time.Second, }, IDCache: config.IDCache{