From 8b688302018c71047f144edefb6d6ac2c5baeb6c Mon Sep 17 00:00:00 2001 From: Andre Duffeck Date: Mon, 6 Mar 2023 15:04:19 +0100 Subject: [PATCH] [full-ci] Improve ini backend, support redis sentinel caches (#5737) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Configure the file metadata cache for storage-users * Also configure the cache for the system storage * Update services/storage-system/pkg/config/config.go Co-authored-by: Martin * Improve documentation * Bump reva to pull in the latest ini backend changes * Fix missing comment * Update services/storage-users/README.md Co-authored-by: Martin * Update services/storage-system/README.md Co-authored-by: Martin * Tweak docs * Apply suggestions from code review Co-authored-by: Jörn Friedrich Dreyer Co-authored-by: Martin --------- Co-authored-by: Martin Co-authored-by: Jörn Friedrich Dreyer --- go.mod | 8 ++++---- go.sum | 15 +++++++------- services/storage-system/README.md | 18 +++++++++++++++++ services/storage-system/pkg/config/config.go | 8 ++++++++ .../pkg/config/defaults/defaultconfig.go | 4 ++++ .../storage-system/pkg/revaconfig/config.go | 5 +++++ services/storage-users/README.md | 15 ++++++++++++++ services/storage-users/pkg/config/config.go | 6 +++--- .../storage-users/pkg/revaconfig/drivers.go | 20 +++++++++++++++++++ 9 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 services/storage-system/README.md diff --git a/go.mod b/go.mod index ff744ea884..796ceddf93 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.5 github.com/coreos/go-oidc/v3 v3.4.0 github.com/cs3org/go-cs3apis v0.0.0-20221012090518-ef2996678965 - github.com/cs3org/reva/v2 v2.12.1-0.20230303141425-83ae3a7e5333 + github.com/cs3org/reva/v2 v2.12.1-0.20230306101206-3de7ec137baf github.com/disintegration/imaging v1.6.2 github.com/ggwhite/go-masker v1.0.9 github.com/go-chi/chi/v5 v5.0.7 @@ -31,7 +31,7 @@ require ( github.com/go-micro/plugins/v4/server/grpc v1.2.0 github.com/go-micro/plugins/v4/server/http v1.2.0 github.com/go-micro/plugins/v4/store/nats-js v1.1.0 - github.com/go-micro/plugins/v4/store/redis v1.1.0 + github.com/go-micro/plugins/v4/store/redis v1.2.0 github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0 github.com/go-micro/plugins/v4/wrapper/monitoring/prometheus v1.2.0 github.com/go-micro/plugins/v4/wrapper/trace/opencensus v1.1.0 @@ -57,7 +57,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/onsi/ginkgo v1.16.5 github.com/onsi/ginkgo/v2 v2.7.0 - github.com/onsi/gomega v1.24.1 + github.com/onsi/gomega v1.25.0 github.com/orcaman/concurrent-map v1.0.0 github.com/owncloud/libre-graph-api-go v1.0.2-0.20230119095249-5688a1b749dc github.com/pkg/errors v0.9.1 @@ -137,7 +137,7 @@ require ( github.com/bombsimon/logrusr/v3 v3.1.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/ceph/go-ceph v0.18.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cevaris/ordered_map v0.0.0-20190319150403-3adeae072e73 // indirect github.com/cloudflare/circl v1.2.0 // indirect github.com/coreos/go-oidc v2.2.1+incompatible // indirect diff --git a/go.sum b/go.sum index 7a4f8ef8cc..322ebfcba1 100644 --- a/go.sum +++ b/go.sum @@ -295,8 +295,9 @@ github.com/ceph/go-ceph v0.18.0 h1:4WM6yAq/iqBDaeeADDiPKLqKiP0iZ4fffdgCr1lnOL4= github.com/ceph/go-ceph v0.18.0/go.mod h1:cflETVTBNAQM6jdr7hpNHHFHKYiJiWWcAeRDrRx/1ng= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cevaris/ordered_map v0.0.0-20190319150403-3adeae072e73 h1:q1g9lSyo/nOIC3W5E3FK3Unrz8b9LdLXCyuC+ZcpPC0= github.com/cevaris/ordered_map v0.0.0-20190319150403-3adeae072e73/go.mod h1:507vXsotcZop7NZfBWdhPmVeOse4ko2R7AagJYrpoEg= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -344,8 +345,8 @@ github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3p github.com/crewjam/saml v0.4.6/go.mod h1:ZBOXnNPFzB3CgOkRm7Nd6IVdkG+l/wF+0ZXLqD96t1A= github.com/crewjam/saml v0.4.9 h1:X2jDv4dv3IvfT9t+RhADavzNFAcq3fVxzTCIH3G605U= github.com/crewjam/saml v0.4.9/go.mod h1:9Zh6dWPtB3MSzTRt8fIFH60Z351QQ+s7hCU3J/tTlA4= -github.com/cs3org/reva/v2 v2.12.1-0.20230303141425-83ae3a7e5333 h1:1z2AmAGfAgIqCE09CIQFpUhMXwG38PlqVYZqKbrxSXQ= -github.com/cs3org/reva/v2 v2.12.1-0.20230303141425-83ae3a7e5333/go.mod h1:dbaNP2U3nGQA5BHLc5w/hqviq7b0F4eygNwC38jeaiU= +github.com/cs3org/reva/v2 v2.12.1-0.20230306101206-3de7ec137baf h1:Qj8tUBW89yglwUx+q2vZN1mIlkrHdd/FVMAN6Wg4T0U= +github.com/cs3org/reva/v2 v2.12.1-0.20230306101206-3de7ec137baf/go.mod h1:EGPAVHVqMHZ6GBo4T6QMw13D0djGU8ZCi/o5siJ9IUs= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= @@ -498,8 +499,8 @@ github.com/go-micro/plugins/v4/server/http v1.2.0 h1:7wgXxdTMSwCfxpUdmOdllyJQRZk github.com/go-micro/plugins/v4/server/http v1.2.0/go.mod h1:YuAjaSPxcn3LI8j2FUsqx0Rxunrj4YwDV41Ax76rLl0= github.com/go-micro/plugins/v4/store/nats-js v1.1.0 h1:6Fe1/eLtg8kRyaGvMILp4olYtTDGwYNBXyb1sYfAWGk= github.com/go-micro/plugins/v4/store/nats-js v1.1.0/go.mod h1:jJf7Gm39OafZlT3s3UE2/9NIYj6OlI2fmZ4czSA3gvo= -github.com/go-micro/plugins/v4/store/redis v1.1.0 h1:QWl5FAfMaekppZGNcj0mSyi8vCkjZGvHoGdaBNeKNm4= -github.com/go-micro/plugins/v4/store/redis v1.1.0/go.mod h1:yXFv4vhmr1ccxRfHxvuSetIBZDOJvRA404PZk5z6/bc= +github.com/go-micro/plugins/v4/store/redis v1.2.0 h1:jR7sHOD1a735cxyBNFif58tP0Ck8OUklpDN1IzzDoRg= +github.com/go-micro/plugins/v4/store/redis v1.2.0/go.mod h1:MbCG0YiyPqETTtm7uHFmxQNCaW1o9hBoYtFwhbVjLUg= github.com/go-micro/plugins/v4/transport/grpc v1.1.0 h1:mXfDYfFQLnVDzjGY3o84oe4prfux9h8txsnA19dKsj8= github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0 h1:EQj4l7fuOSz8ueUYhFlpZPp9+tN4JeONL32ARRKXW/U= github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0/go.mod h1:JR9Ox/iJIrcXm8nCWdAEBsyG7Q7lyMLzsTZPfXrqvwo= @@ -1051,8 +1052,8 @@ github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1L github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= +github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= diff --git a/services/storage-system/README.md b/services/storage-system/README.md new file mode 100644 index 0000000000..5cdb8cb44a --- /dev/null +++ b/services/storage-system/README.md @@ -0,0 +1,18 @@ +# Storage-System Service + +Purpose and description to be added + +## Caching + +The `storage-system` service caches file metadata via the configured store in `STORAGE_SYSTEM_CACHE_STORE`. Possible stores are: + - `memory`: Basic in-memory store and the default. + - `redis`: Stores metadata in a configured Redis cluster. + - `redis-sentinel`: Stores metadata in a configured Redis Sentinel cluster. + - `etcd`: Stores metadata in a configured etcd cluster. + - `nats-js`: Stores metadata using the key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store) + - `noop`: Stores nothing. Useful for testing. Not recommended in productive enviroments. + +1. Note that in-memory stores are by nature not reboot persistent. +2. Though usually not necessary, a database name can be configured for event stores if the event store supports this. Generally not applicapable for stores of type `in-memory`, `redis` and `redis-sentinel`. These settings are blank by default which means that the standard settings of the configured store applies. +3. The `storage-system` service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances. +4. When using `redis-sentinel`, the Redis master to use is configured via `STORAGE_SYSTEM_CACHE_NODES` in the form of `:/` like `10.10.0.200:26379/mymaster`. \ No newline at end of file diff --git a/services/storage-system/pkg/config/config.go b/services/storage-system/pkg/config/config.go index e1282f751f..17ed2dd8dc 100644 --- a/services/storage-system/pkg/config/config.go +++ b/services/storage-system/pkg/config/config.go @@ -23,6 +23,7 @@ type Config struct { SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_SYSTEM_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."` + Cache Cache `yaml:"cache"` Driver string `yaml:"driver" env:"STORAGE_SYSTEM_DRIVER" desc:"The driver which should be used by the service."` Drivers Drivers `yaml:"drivers"` DataServerURL string `yaml:"data_server_url" env:"STORAGE_SYSTEM_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by other services using this service."` @@ -80,3 +81,10 @@ type OCISDriver struct { MaxAcquireLockCycles int `yaml:"max_acquire_lock_cycles" env:"STORAGE_SYSTEM_OCIS_MAX_ACQUIRE_LOCK_CYCLES" desc:"When trying to lock files, ocis will try this amount of times to acquire the lock before failing. After each try it will wait for an increasing amount of time. Values of 0 or below will be ignored and the default value of 20 will be used."` LockCycleDurationFactor int `yaml:"lock_cycle_duration_factor" env:"STORAGE_SYSTEM_OCIS_LOCK_CYCLE_DURATION_FACTOR" desc:"When trying to lock files, ocis will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value of 30 will be used."` } + +// Cache holds cache config +type Cache struct { + Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_SYSTEM_CACHE_STORE" desc:"Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', 'etcd' and 'noop'. See the text description for details."` + Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_SYSTEM_CACHE_NODES" desc:"A comma separated list of addresses to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."` + Database string `yaml:"database" env:"STORAGE_SYSTEM_CACHE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when a 'memory', 'redis' or 'redis-sentinel' store is configured."` +} diff --git a/services/storage-system/pkg/config/defaults/defaultconfig.go b/services/storage-system/pkg/config/defaults/defaultconfig.go index 06b85be28f..01a1a459ec 100644 --- a/services/storage-system/pkg/config/defaults/defaultconfig.go +++ b/services/storage-system/pkg/config/defaults/defaultconfig.go @@ -47,6 +47,10 @@ func DefaultConfig() *config.Config { LockCycleDurationFactor: 30, }, }, + Cache: config.Cache{ + Store: "memory", + Database: "system", + }, } } diff --git a/services/storage-system/pkg/revaconfig/config.go b/services/storage-system/pkg/revaconfig/config.go index 007280c824..7d597c4ab3 100644 --- a/services/storage-system/pkg/revaconfig/config.go +++ b/services/storage-system/pkg/revaconfig/config.go @@ -161,6 +161,11 @@ func metadataDrivers(cfg *config.Config) map[string]interface{} { "permissionssvc": cfg.GRPC.Addr, "max_acquire_lock_cycles": cfg.Drivers.OCIS.MaxAcquireLockCycles, "lock_cycle_duration_factor": cfg.Drivers.OCIS.LockCycleDurationFactor, + "filemetadatacache": map[string]interface{}{ + "cache_store": cfg.Cache.Store, + "cache_nodes": cfg.Cache.Nodes, + "cache_database": cfg.Cache.Database, + }, }, } } diff --git a/services/storage-users/README.md b/services/storage-users/README.md index e020087a42..5e36463e83 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -80,3 +80,18 @@ The configuration for the `purge-expired` command is done by using the following * `STORAGE_USERS_PURGE_TRASH_BIN_PERSONAL_DELETE_BEFORE` has a default value of `30 days`, which means the command will delete all files older than `30 days`. The value is human-readable, valid values are `24h`, `60m`, `60s` etc. `0` is equivalent to disable and prevents the deletion of `personal space` trash-bin files. * `STORAGE_USERS_PURGE_TRASH_BIN_PROJECT_DELETE_BEFORE` has a default value of `30 days`, which means the command will delete all files older than `30 days`. The value is human-readable, valid values are `24h`, `60m`, `60s` etc. `0` is equivalent to disable and prevents the deletion of `project space` trash-bin files. + +## Caching + +The `storage-users` service caches file metadata via the configured store in `STORAGE_USERS_CACHE_STORE`. Possible stores are: + - `memory`: Basic in-memory store and the default. + - `redis`: Stores metadata in a configured Redis cluster. + - `redis-sentinel`: Stores metadata in a configured Redis Sentinel cluster. + - `etcd`: Stores metadata in a configured etcd cluster. + - `nats-js`: Stores metadata using the key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store) + - `noop`: Stores nothing. Useful for testing. Not recommended in productive enviroments. + +1. Note that in-memory stores are by nature not reboot persistent. +2. Though usually not necessary, a database name can be configured for event stores if the event store supports this. Generally not applicapable for stores of type `in-memory`, `redis` and `redis-sentinel`. These settings are blank by default which means that the standard settings of the configured store applies. +3. The `storage-users` service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances. +4. When using `redis-sentinel`, the Redis master to use is configured via `STORAGE_SYSTEM_CACHE_NODES` in the form of `:/` like `10.10.0.200:26379/mymaster`. \ No newline at end of file diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 8efd4439e6..0a1655af2a 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -169,9 +169,9 @@ type Events struct { // Cache holds cache config type Cache struct { - Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE" desc:"Store implementation for the cache. Valid values are \"memory\" (default), \"redis\", and \"etcd\"."` - Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_NODES" desc:"Node addresses to use for the cache store."` - Database string `yaml:"database" env:"STORAGE_USERS_CACHE_DATABASE" desc:"Database name of the cache."` + Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE" desc:"Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', and 'etcd'. See the text description for details."` + Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_NODES" desc:"A comma separated list of addresses to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."` + Database string `yaml:"database" env:"STORAGE_USERS_CACHE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when a 'memory', 'redis' or 'redis-sentinel' store is configured."` } // S3Driver is the storage driver configuration when using 's3' storage driver diff --git a/services/storage-users/pkg/revaconfig/drivers.go b/services/storage-users/pkg/revaconfig/drivers.go index 9a9ec41623..6a641da928 100644 --- a/services/storage-users/pkg/revaconfig/drivers.go +++ b/services/storage-users/pkg/revaconfig/drivers.go @@ -130,6 +130,11 @@ func Ocis(cfg *config.Config) map[string]interface{} { "cache_nodes": cfg.Cache.Nodes, "cache_database": cfg.Cache.Database, }, + "filemetadatacache": map[string]interface{}{ + "cache_store": cfg.Cache.Store, + "cache_nodes": cfg.Cache.Nodes, + "cache_database": cfg.Cache.Database, + }, "events": map[string]interface{}{ "natsaddress": cfg.Events.Addr, "natsclusterid": cfg.Events.ClusterID, @@ -167,6 +172,11 @@ func OcisNoEvents(cfg *config.Config) map[string]interface{} { "cache_nodes": cfg.Cache.Nodes, "cache_database": cfg.Cache.Database, }, + "filemetadatacache": map[string]interface{}{ + "cache_store": cfg.Cache.Store, + "cache_nodes": cfg.Cache.Nodes, + "cache_database": cfg.Cache.Database, + }, } } @@ -209,6 +219,11 @@ func S3NG(cfg *config.Config) map[string]interface{} { "cache_nodes": cfg.Cache.Nodes, "cache_database": cfg.Cache.Database, }, + "filemetadatacache": map[string]interface{}{ + "cache_store": cfg.Cache.Store, + "cache_nodes": cfg.Cache.Nodes, + "cache_database": cfg.Cache.Database, + }, "events": map[string]interface{}{ "natsaddress": cfg.Events.Addr, "natsclusterid": cfg.Events.ClusterID, @@ -250,5 +265,10 @@ func S3NGNoEvents(cfg *config.Config) map[string]interface{} { "cache_nodes": cfg.Cache.Nodes, "cache_database": cfg.Cache.Database, }, + "filemetadatacache": map[string]interface{}{ + "cache_store": cfg.Cache.Store, + "cache_nodes": cfg.Cache.Nodes, + "cache_database": cfg.Cache.Database, + }, } }