mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-02-25 15:09:27 -06:00
[full-ci] Improve ini backend, support redis sentinel caches (#5737)
* Configure the file metadata cache for storage-users * Also configure the cache for the system storage * Update services/storage-system/pkg/config/config.go Co-authored-by: Martin <github@diemattels.at> * Improve documentation * Bump reva to pull in the latest ini backend changes * Fix missing comment * Update services/storage-users/README.md Co-authored-by: Martin <github@diemattels.at> * Update services/storage-system/README.md Co-authored-by: Martin <github@diemattels.at> * Tweak docs * Apply suggestions from code review Co-authored-by: Jörn Friedrich Dreyer <jfd@butonic.de> Co-authored-by: Martin <github@diemattels.at> --------- Co-authored-by: Martin <github@diemattels.at> Co-authored-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
Andre Duffeck
8
go.mod
8
go.mod
@@ -11,7 +11,7 @@ require (
|
||||
github.com/blevesearch/bleve/v2 v2.3.5
|
||||
github.com/coreos/go-oidc/v3 v3.4.0
|
||||
github.com/cs3org/go-cs3apis v0.0.0-20221012090518-ef2996678965
|
||||
github.com/cs3org/reva/v2 v2.12.1-0.20230303141425-83ae3a7e5333
|
||||
github.com/cs3org/reva/v2 v2.12.1-0.20230306101206-3de7ec137baf
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/ggwhite/go-masker v1.0.9
|
||||
github.com/go-chi/chi/v5 v5.0.7
|
||||
@@ -31,7 +31,7 @@ require (
|
||||
github.com/go-micro/plugins/v4/server/grpc v1.2.0
|
||||
github.com/go-micro/plugins/v4/server/http v1.2.0
|
||||
github.com/go-micro/plugins/v4/store/nats-js v1.1.0
|
||||
github.com/go-micro/plugins/v4/store/redis v1.1.0
|
||||
github.com/go-micro/plugins/v4/store/redis v1.2.0
|
||||
github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0
|
||||
github.com/go-micro/plugins/v4/wrapper/monitoring/prometheus v1.2.0
|
||||
github.com/go-micro/plugins/v4/wrapper/trace/opencensus v1.1.0
|
||||
@@ -57,7 +57,7 @@ require (
|
||||
github.com/olekukonko/tablewriter v0.0.5
|
||||
github.com/onsi/ginkgo v1.16.5
|
||||
github.com/onsi/ginkgo/v2 v2.7.0
|
||||
github.com/onsi/gomega v1.24.1
|
||||
github.com/onsi/gomega v1.25.0
|
||||
github.com/orcaman/concurrent-map v1.0.0
|
||||
github.com/owncloud/libre-graph-api-go v1.0.2-0.20230119095249-5688a1b749dc
|
||||
github.com/pkg/errors v0.9.1
|
||||
@@ -137,7 +137,7 @@ require (
|
||||
github.com/bombsimon/logrusr/v3 v3.1.0 // indirect
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
|
||||
github.com/ceph/go-ceph v0.18.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cevaris/ordered_map v0.0.0-20190319150403-3adeae072e73 // indirect
|
||||
github.com/cloudflare/circl v1.2.0 // indirect
|
||||
github.com/coreos/go-oidc v2.2.1+incompatible // indirect
|
||||
|
||||
15
go.sum
15
go.sum
@@ -295,8 +295,9 @@ github.com/ceph/go-ceph v0.18.0 h1:4WM6yAq/iqBDaeeADDiPKLqKiP0iZ4fffdgCr1lnOL4=
|
||||
github.com/ceph/go-ceph v0.18.0/go.mod h1:cflETVTBNAQM6jdr7hpNHHFHKYiJiWWcAeRDrRx/1ng=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cevaris/ordered_map v0.0.0-20190319150403-3adeae072e73 h1:q1g9lSyo/nOIC3W5E3FK3Unrz8b9LdLXCyuC+ZcpPC0=
|
||||
github.com/cevaris/ordered_map v0.0.0-20190319150403-3adeae072e73/go.mod h1:507vXsotcZop7NZfBWdhPmVeOse4ko2R7AagJYrpoEg=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
@@ -344,8 +345,8 @@ github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3p
|
||||
github.com/crewjam/saml v0.4.6/go.mod h1:ZBOXnNPFzB3CgOkRm7Nd6IVdkG+l/wF+0ZXLqD96t1A=
|
||||
github.com/crewjam/saml v0.4.9 h1:X2jDv4dv3IvfT9t+RhADavzNFAcq3fVxzTCIH3G605U=
|
||||
github.com/crewjam/saml v0.4.9/go.mod h1:9Zh6dWPtB3MSzTRt8fIFH60Z351QQ+s7hCU3J/tTlA4=
|
||||
github.com/cs3org/reva/v2 v2.12.1-0.20230303141425-83ae3a7e5333 h1:1z2AmAGfAgIqCE09CIQFpUhMXwG38PlqVYZqKbrxSXQ=
|
||||
github.com/cs3org/reva/v2 v2.12.1-0.20230303141425-83ae3a7e5333/go.mod h1:dbaNP2U3nGQA5BHLc5w/hqviq7b0F4eygNwC38jeaiU=
|
||||
github.com/cs3org/reva/v2 v2.12.1-0.20230306101206-3de7ec137baf h1:Qj8tUBW89yglwUx+q2vZN1mIlkrHdd/FVMAN6Wg4T0U=
|
||||
github.com/cs3org/reva/v2 v2.12.1-0.20230306101206-3de7ec137baf/go.mod h1:EGPAVHVqMHZ6GBo4T6QMw13D0djGU8ZCi/o5siJ9IUs=
|
||||
github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
|
||||
github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
|
||||
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
|
||||
@@ -498,8 +499,8 @@ github.com/go-micro/plugins/v4/server/http v1.2.0 h1:7wgXxdTMSwCfxpUdmOdllyJQRZk
|
||||
github.com/go-micro/plugins/v4/server/http v1.2.0/go.mod h1:YuAjaSPxcn3LI8j2FUsqx0Rxunrj4YwDV41Ax76rLl0=
|
||||
github.com/go-micro/plugins/v4/store/nats-js v1.1.0 h1:6Fe1/eLtg8kRyaGvMILp4olYtTDGwYNBXyb1sYfAWGk=
|
||||
github.com/go-micro/plugins/v4/store/nats-js v1.1.0/go.mod h1:jJf7Gm39OafZlT3s3UE2/9NIYj6OlI2fmZ4czSA3gvo=
|
||||
github.com/go-micro/plugins/v4/store/redis v1.1.0 h1:QWl5FAfMaekppZGNcj0mSyi8vCkjZGvHoGdaBNeKNm4=
|
||||
github.com/go-micro/plugins/v4/store/redis v1.1.0/go.mod h1:yXFv4vhmr1ccxRfHxvuSetIBZDOJvRA404PZk5z6/bc=
|
||||
github.com/go-micro/plugins/v4/store/redis v1.2.0 h1:jR7sHOD1a735cxyBNFif58tP0Ck8OUklpDN1IzzDoRg=
|
||||
github.com/go-micro/plugins/v4/store/redis v1.2.0/go.mod h1:MbCG0YiyPqETTtm7uHFmxQNCaW1o9hBoYtFwhbVjLUg=
|
||||
github.com/go-micro/plugins/v4/transport/grpc v1.1.0 h1:mXfDYfFQLnVDzjGY3o84oe4prfux9h8txsnA19dKsj8=
|
||||
github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0 h1:EQj4l7fuOSz8ueUYhFlpZPp9+tN4JeONL32ARRKXW/U=
|
||||
github.com/go-micro/plugins/v4/wrapper/breaker/gobreaker v1.2.0/go.mod h1:JR9Ox/iJIrcXm8nCWdAEBsyG7Q7lyMLzsTZPfXrqvwo=
|
||||
@@ -1051,8 +1052,8 @@ github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1L
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E=
|
||||
github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
|
||||
github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
|
||||
github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
|
||||
|
||||
18
services/storage-system/README.md
Normal file
18
services/storage-system/README.md
Normal file
@@ -0,0 +1,18 @@
|
||||
# Storage-System Service
|
||||
|
||||
Purpose and description to be added
|
||||
|
||||
## Caching
|
||||
|
||||
The `storage-system` service caches file metadata via the configured store in `STORAGE_SYSTEM_CACHE_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `redis`: Stores metadata in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores metadata in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores metadata in a configured etcd cluster.
|
||||
- `nats-js`: Stores metadata using the key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in productive enviroments.
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot persistent.
|
||||
2. Though usually not necessary, a database name can be configured for event stores if the event store supports this. Generally not applicapable for stores of type `in-memory`, `redis` and `redis-sentinel`. These settings are blank by default which means that the standard settings of the configured store applies.
|
||||
3. The `storage-system` service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `STORAGE_SYSTEM_CACHE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
@@ -23,6 +23,7 @@ type Config struct {
|
||||
|
||||
SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_SYSTEM_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."`
|
||||
|
||||
Cache Cache `yaml:"cache"`
|
||||
Driver string `yaml:"driver" env:"STORAGE_SYSTEM_DRIVER" desc:"The driver which should be used by the service."`
|
||||
Drivers Drivers `yaml:"drivers"`
|
||||
DataServerURL string `yaml:"data_server_url" env:"STORAGE_SYSTEM_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by other services using this service."`
|
||||
@@ -80,3 +81,10 @@ type OCISDriver struct {
|
||||
MaxAcquireLockCycles int `yaml:"max_acquire_lock_cycles" env:"STORAGE_SYSTEM_OCIS_MAX_ACQUIRE_LOCK_CYCLES" desc:"When trying to lock files, ocis will try this amount of times to acquire the lock before failing. After each try it will wait for an increasing amount of time. Values of 0 or below will be ignored and the default value of 20 will be used."`
|
||||
LockCycleDurationFactor int `yaml:"lock_cycle_duration_factor" env:"STORAGE_SYSTEM_OCIS_LOCK_CYCLE_DURATION_FACTOR" desc:"When trying to lock files, ocis will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value of 30 will be used."`
|
||||
}
|
||||
|
||||
// Cache holds cache config
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_SYSTEM_CACHE_STORE" desc:"Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', 'etcd' and 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_SYSTEM_CACHE_NODES" desc:"A comma separated list of addresses to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"STORAGE_SYSTEM_CACHE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when a 'memory', 'redis' or 'redis-sentinel' store is configured."`
|
||||
}
|
||||
|
||||
@@ -47,6 +47,10 @@ func DefaultConfig() *config.Config {
|
||||
LockCycleDurationFactor: 30,
|
||||
},
|
||||
},
|
||||
Cache: config.Cache{
|
||||
Store: "memory",
|
||||
Database: "system",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -161,6 +161,11 @@ func metadataDrivers(cfg *config.Config) map[string]interface{} {
|
||||
"permissionssvc": cfg.GRPC.Addr,
|
||||
"max_acquire_lock_cycles": cfg.Drivers.OCIS.MaxAcquireLockCycles,
|
||||
"lock_cycle_duration_factor": cfg.Drivers.OCIS.LockCycleDurationFactor,
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.Store,
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,3 +80,18 @@ The configuration for the `purge-expired` command is done by using the following
|
||||
* `STORAGE_USERS_PURGE_TRASH_BIN_PERSONAL_DELETE_BEFORE` has a default value of `30 days`, which means the command will delete all files older than `30 days`. The value is human-readable, valid values are `24h`, `60m`, `60s` etc. `0` is equivalent to disable and prevents the deletion of `personal space` trash-bin files.
|
||||
|
||||
* `STORAGE_USERS_PURGE_TRASH_BIN_PROJECT_DELETE_BEFORE` has a default value of `30 days`, which means the command will delete all files older than `30 days`. The value is human-readable, valid values are `24h`, `60m`, `60s` etc. `0` is equivalent to disable and prevents the deletion of `project space` trash-bin files.
|
||||
|
||||
## Caching
|
||||
|
||||
The `storage-users` service caches file metadata via the configured store in `STORAGE_USERS_CACHE_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `redis`: Stores metadata in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores metadata in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores metadata in a configured etcd cluster.
|
||||
- `nats-js`: Stores metadata using the key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in productive enviroments.
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot persistent.
|
||||
2. Though usually not necessary, a database name can be configured for event stores if the event store supports this. Generally not applicapable for stores of type `in-memory`, `redis` and `redis-sentinel`. These settings are blank by default which means that the standard settings of the configured store applies.
|
||||
3. The `storage-users` service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `STORAGE_SYSTEM_CACHE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
@@ -169,9 +169,9 @@ type Events struct {
|
||||
|
||||
// Cache holds cache config
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE" desc:"Store implementation for the cache. Valid values are \"memory\" (default), \"redis\", and \"etcd\"."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_NODES" desc:"Node addresses to use for the cache store."`
|
||||
Database string `yaml:"database" env:"STORAGE_USERS_CACHE_DATABASE" desc:"Database name of the cache."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE" desc:"Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', and 'etcd'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_NODES" desc:"A comma separated list of addresses to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"STORAGE_USERS_CACHE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when a 'memory', 'redis' or 'redis-sentinel' store is configured."`
|
||||
}
|
||||
|
||||
// S3Driver is the storage driver configuration when using 's3' storage driver
|
||||
|
||||
@@ -130,6 +130,11 @@ func Ocis(cfg *config.Config) map[string]interface{} {
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.Store,
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"events": map[string]interface{}{
|
||||
"natsaddress": cfg.Events.Addr,
|
||||
"natsclusterid": cfg.Events.ClusterID,
|
||||
@@ -167,6 +172,11 @@ func OcisNoEvents(cfg *config.Config) map[string]interface{} {
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.Store,
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,6 +219,11 @@ func S3NG(cfg *config.Config) map[string]interface{} {
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.Store,
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"events": map[string]interface{}{
|
||||
"natsaddress": cfg.Events.Addr,
|
||||
"natsclusterid": cfg.Events.ClusterID,
|
||||
@@ -250,5 +265,10 @@ func S3NGNoEvents(cfg *config.Config) map[string]interface{} {
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.Store,
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user