mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-01-04 19:29:49 -06:00
Merge pull request #7979 from kobergj/InmemNatsjsStore
[full-ci] Add inmem option for nats-js stores
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
Enhancement: Allow inmemory nats-js-kv stores
|
||||
|
||||
Adds envvars to keep nats-js-kv stores in memory and not persist them on disc.
|
||||
|
||||
https://github.com/owncloud/ocis/pull/7979
|
||||
@@ -4,3 +4,4 @@ Bumps reva version
|
||||
|
||||
https://github.com/owncloud/ocis/pull/7793
|
||||
https://github.com/owncloud/ocis/pull/7978
|
||||
https://github.com/owncloud/ocis/pull/7979
|
||||
|
||||
2
go.mod
2
go.mod
@@ -13,7 +13,7 @@ require (
|
||||
github.com/coreos/go-oidc v2.2.1+incompatible
|
||||
github.com/coreos/go-oidc/v3 v3.9.0
|
||||
github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781
|
||||
github.com/cs3org/reva/v2 v2.17.1-0.20231215110403-2f85376a4d3f
|
||||
github.com/cs3org/reva/v2 v2.17.1-0.20231215113433-48c0ea55bf47
|
||||
github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
|
||||
|
||||
6
go.sum
6
go.sum
@@ -1021,8 +1021,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c=
|
||||
github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME=
|
||||
github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY=
|
||||
github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
|
||||
github.com/cs3org/reva/v2 v2.17.1-0.20231215110403-2f85376a4d3f h1:AF7fcQ7dfi3sxKLpIFJ8Y2uKJ8umB9K+nFe+3tDa4oA=
|
||||
github.com/cs3org/reva/v2 v2.17.1-0.20231215110403-2f85376a4d3f/go.mod h1:oX1YtLKGr7jatGk0CpPM4GKbSEIdHhmsQuSAYElnN1U=
|
||||
github.com/cs3org/reva/v2 v2.17.1-0.20231215113433-48c0ea55bf47 h1:6DfMeFpCXoqlfm/+FJ/mFs8Ul5WCZNlorsbDM9Z/ATE=
|
||||
github.com/cs3org/reva/v2 v2.17.1-0.20231215113433-48c0ea55bf47/go.mod h1:oX1YtLKGr7jatGk0CpPM4GKbSEIdHhmsQuSAYElnN1U=
|
||||
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
@@ -1688,8 +1688,6 @@ github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b h1:Q53idHrTuQD
|
||||
github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b/go.mod h1:KirJrATYGbTyUwVR26xIkaipRqRcMRXBf8N5dacvGus=
|
||||
github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103 h1:Z/i1e+gTZrmcGeZyWckaLfucYG6KYOXLWo4co8pZYNY=
|
||||
github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103/go.mod h1:o9YPB5aGP8ob35Vy6+vyq3P3bWe7NQWzf+JLiXCiMaE=
|
||||
github.com/micbar/reva/v2 v2.0.0-20231214215940-ac484c63b45e h1:IsCx3d+WflZgIGFV5TPqGoMRWiygaj11QJvefcUNTUQ=
|
||||
github.com/micbar/reva/v2 v2.0.0-20231214215940-ac484c63b45e/go.mod h1:oX1YtLKGr7jatGk0CpPM4GKbSEIdHhmsQuSAYElnN1U=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
||||
github.com/miekg/dns v1.1.40/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
|
||||
@@ -56,12 +56,13 @@ type HTTPServiceTLS struct {
|
||||
}
|
||||
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"OCIS_CACHE_STORE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"OCIS_CACHE_STORE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured."`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
}
|
||||
|
||||
// Commons holds configuration that are common to all extensions. Each extension can then decide whether
|
||||
|
||||
@@ -14,17 +14,24 @@ The `eventhistory` services consumes all events from the configured event system
|
||||
|
||||
The `eventhistory` service stores each consumed event via the configured store in `EVENTHISTORY_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `ocmem`: Advanced in-memory store allowing max size.
|
||||
- `redis`: Stores data in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores data in a configured etcd cluster.
|
||||
- `nats-js`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
- `ocmem`: Advanced in-memory store allowing max size. (deprecated)
|
||||
- `redis`: Stores data in a configured Redis cluster. (deprecated)
|
||||
- `etcd`: Stores data in a configured etcd cluster. (deprecated)
|
||||
- `nats-js`: Stores data using object-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/obj_store) (deprecated)
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot-persistent.
|
||||
2. Though usually not necessary, a database name and a database table can be configured for event stores if the event store supports this. Generally not applicable for stores of type `in-memory`. These settings are blank by default which means that the standard settings of the configured store apply.
|
||||
4. The eventhistory service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
5. When using `redis-sentinel`, the Redis master to use is configured via `EVENTHISTORY_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
## Retrieving
|
||||
|
||||
|
||||
@@ -44,17 +44,24 @@ You can find more details regarding available attributes at the [libre-graph-api
|
||||
|
||||
The `frontend` service can use a configured store via `FRONTEND_OCS_STAT_CACHE_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `ocmem`: Advanced in-memory store allowing max size.
|
||||
- `redis`: Stores data in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores data in a configured etcd cluster.
|
||||
- `nats-js`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
- `ocmem`: Advanced in-memory store allowing max size. (deprecated)
|
||||
- `redis`: Stores data in a configured Redis cluster. (deprecated)
|
||||
- `etcd`: Stores data in a configured etcd cluster. (deprecated)
|
||||
- `nats-js`: Stores data using object-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/obj_store) (deprecated)
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot-persistent.
|
||||
2. Though usually not necessary, a database name and a database table can be configured for event stores if the event store supports this. Generally not applicable for stores of type `in-memory`. These settings are blank by default which means that the standard settings of the configured store apply.
|
||||
3. The frontend service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `FRONTEND_OCS_STAT_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
## Event Handler
|
||||
|
||||
|
||||
@@ -126,16 +126,18 @@ type DataGateway struct {
|
||||
}
|
||||
|
||||
type OCS struct {
|
||||
Prefix string `yaml:"prefix" env:"FRONTEND_OCS_PREFIX" desc:"URL path prefix for the OCS service. Note that the string must not start with '/'."`
|
||||
SharePrefix string `yaml:"share_prefix" env:"FRONTEND_OCS_SHARE_PREFIX" desc:"Path prefix for shares as part of an ocis resource. Note that the path must start with '/'."`
|
||||
HomeNamespace string `yaml:"home_namespace" env:"FRONTEND_OCS_PERSONAL_NAMESPACE" desc:"Homespace namespace identifier."`
|
||||
AdditionalInfoAttribute string `yaml:"additional_info_attribute" env:"FRONTEND_OCS_ADDITIONAL_INFO_ATTRIBUTE" desc:"Additional information attribute for the user like {{.Mail}}."`
|
||||
StatCacheType string `yaml:"stat_cache_type" env:"OCIS_CACHE_STORE;FRONTEND_OCS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;FRONTEND_OCS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
StatCacheTable string `yaml:"stat_cache_table" env:"FRONTEND_OCS_STAT_CACHE_TABLE" desc:"The database table the store should use."`
|
||||
StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;FRONTEND_OCS_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;FRONTEND_OCS_STAT_CACHE_SIZE" desc:"Max number of entries to hold in the cache."`
|
||||
Prefix string `yaml:"prefix" env:"FRONTEND_OCS_PREFIX" desc:"URL path prefix for the OCS service. Note that the string must not start with '/'."`
|
||||
SharePrefix string `yaml:"share_prefix" env:"FRONTEND_OCS_SHARE_PREFIX" desc:"Path prefix for shares as part of an ocis resource. Note that the path must start with '/'."`
|
||||
HomeNamespace string `yaml:"home_namespace" env:"FRONTEND_OCS_PERSONAL_NAMESPACE" desc:"Homespace namespace identifier."`
|
||||
AdditionalInfoAttribute string `yaml:"additional_info_attribute" env:"FRONTEND_OCS_ADDITIONAL_INFO_ATTRIBUTE" desc:"Additional information attribute for the user like {{.Mail}}."`
|
||||
StatCacheType string `yaml:"stat_cache_type" env:"OCIS_CACHE_STORE;FRONTEND_OCS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;FRONTEND_OCS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
StatCacheTable string `yaml:"stat_cache_table" env:"FRONTEND_OCS_STAT_CACHE_TABLE" desc:"The database table the store should use."`
|
||||
StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;FRONTEND_OCS_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;FRONTEND_OCS_STAT_CACHE_SIZE" desc:"Max number of entries to hold in the cache."`
|
||||
StatCacheDisablePersistence bool `yaml:"stat_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;FRONTEND_OCS_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disable persistence of the cache. Only applies when using the 'nats-js-kv' store type. Defaults to false."`
|
||||
|
||||
CacheWarmupDriver string `yaml:"cache_warmup_driver,omitempty"` // not supported by the oCIS product, therefore not part of docs
|
||||
CacheWarmupDrivers CacheWarmupDrivers `yaml:"cache_warmup_drivers,omitempty"` // not supported by the oCIS product, therefore not part of docs
|
||||
EnableDenials bool `yaml:"enable_denials" env:"FRONTEND_OCS_ENABLE_DENIALS" desc:"EXPERIMENTAL: enable the feature to deny access on folders."`
|
||||
|
||||
@@ -156,15 +156,18 @@ func FrontendConfigFromStruct(cfg *config.Config, logger log.Logger) (map[string
|
||||
"insecure": true,
|
||||
},
|
||||
"ocs": map[string]interface{}{
|
||||
"storage_registry_svc": cfg.Reva.Address,
|
||||
"share_prefix": cfg.OCS.SharePrefix,
|
||||
"home_namespace": cfg.OCS.HomeNamespace,
|
||||
"stat_cache_ttl": cfg.OCS.StatCacheTTL / time.Second,
|
||||
"stat_cache_size": cfg.OCS.StatCacheSize,
|
||||
"stat_cache_store": cfg.OCS.StatCacheType,
|
||||
"stat_cache_nodes": cfg.OCS.StatCacheNodes,
|
||||
"stat_cache_database": cfg.OCS.StatCacheDatabase,
|
||||
"stat_cache_table": cfg.OCS.StatCacheTable,
|
||||
"storage_registry_svc": cfg.Reva.Address,
|
||||
"share_prefix": cfg.OCS.SharePrefix,
|
||||
"home_namespace": cfg.OCS.HomeNamespace,
|
||||
"stat_cache_config": map[string]interface{}{
|
||||
"cache_store": cfg.OCS.StatCacheType,
|
||||
"cache_nodes": cfg.OCS.StatCacheNodes,
|
||||
"cache_database": cfg.OCS.StatCacheDatabase,
|
||||
"cache_table": cfg.OCS.StatCacheTable,
|
||||
"cache_ttl": cfg.OCS.StatCacheTTL / time.Second,
|
||||
"cache_size": cfg.OCS.StatCacheSize,
|
||||
"cache_disable_persistence": cfg.OCS.StatCacheDisablePersistence,
|
||||
},
|
||||
"prefix": cfg.OCS.Prefix,
|
||||
"additional_info_attribute": cfg.OCS.AdditionalInfoAttribute,
|
||||
"machine_auth_apikey": cfg.MachineAuthAPIKey,
|
||||
|
||||
@@ -15,11 +15,18 @@ Use `OCIS_CACHE_STORE` (`GATEWAY_PROVIDER_CACHE_STORE`, `GATEWAY_CREATE_HOME_CAC
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
- `ocmem`: Advanced in-memory store allowing max size. (deprecated)
|
||||
- `redis`: Stores data in a configured Redis cluster. (deprecated)
|
||||
- `etcd`: Stores data in a configured etcd cluster. (deprecated)
|
||||
- `nats-js`: Stores data using object-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/obj_store) (deprecated)
|
||||
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The gateway service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
@@ -85,19 +85,22 @@ type StorageRegistry struct {
|
||||
|
||||
// Cache holds cache config
|
||||
type Cache struct {
|
||||
StatCacheStore string // NOTE: The stat cache is not working atm. Hence we block configuring it
|
||||
StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
ProviderCacheSize int `yaml:"provider_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_PROVIDER_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
CreateHomeCacheSize int `yaml:"create_home_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_CREATE_HOME_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
StatCacheStore string // NOTE: The stat cache is not working atm. Hence we block configuring it
|
||||
StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
StatCacheDisablePersistence bool `yaml:"stat_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the stat cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
ProviderCacheSize int `yaml:"provider_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_PROVIDER_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
ProviderCacheDisablePersistence bool `yaml:"provider_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_PROVIDER_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the provider cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
CreateHomeCacheSize int `yaml:"create_home_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_CREATE_HOME_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
CreateHomeCacheDisablePersistence bool `yaml:"create_home_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_CREATE_HOME_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the create home cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/utils"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
@@ -62,21 +61,33 @@ func GatewayConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]i
|
||||
"transfer_shared_secret": cfg.TransferSecret,
|
||||
"transfer_expires": cfg.TransferExpires,
|
||||
// cache and TTLs
|
||||
"stat_cache_store": cfg.Cache.StatCacheStore,
|
||||
"stat_cache_nodes": cfg.Cache.StatCacheNodes,
|
||||
"stat_cache_database": cfg.Cache.StatCacheDatabase,
|
||||
"stat_cache_ttl": cfg.Cache.StatCacheTTL / time.Second,
|
||||
"stat_cache_size": cfg.Cache.StatCacheSize,
|
||||
"provider_cache_store": cfg.Cache.ProviderCacheStore,
|
||||
"provider_cache_nodes": cfg.Cache.ProviderCacheNodes,
|
||||
"provider_cache_database": cfg.Cache.ProviderCacheDatabase,
|
||||
"provider_cache_ttl": cfg.Cache.ProviderCacheTTL / time.Second,
|
||||
"provider_cache_size": cfg.Cache.ProviderCacheSize,
|
||||
"create_home_cache_store": cfg.Cache.CreateHomeCacheStore,
|
||||
"create_home_cache_nodes": cfg.Cache.CreateHomeCacheNodes,
|
||||
"create_home_cache_database": cfg.Cache.CreateHomeCacheDatabase,
|
||||
"create_home_cache_ttl": cfg.Cache.CreateHomeCacheTTL / time.Second,
|
||||
"create_home_cache_size": cfg.Cache.CreateHomeCacheSize,
|
||||
"stat_cache_config": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.StatCacheStore,
|
||||
"cache_nodes": cfg.Cache.StatCacheNodes,
|
||||
"cache_database": cfg.Cache.StatCacheDatabase,
|
||||
"cache_table": "stat",
|
||||
"cache_ttl": cfg.Cache.StatCacheTTL,
|
||||
"cache_size": cfg.Cache.StatCacheSize,
|
||||
"cache_disable_persistenc": cfg.Cache.StatCacheDisablePersistence,
|
||||
},
|
||||
"provider_cache_config": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.ProviderCacheStore,
|
||||
"cache_nodes": cfg.Cache.ProviderCacheNodes,
|
||||
"cache_database": cfg.Cache.ProviderCacheDatabase,
|
||||
"cache_table": "provider",
|
||||
"cache_ttl": cfg.Cache.ProviderCacheTTL,
|
||||
"cache_size": cfg.Cache.ProviderCacheSize,
|
||||
"disable_persistence": cfg.Cache.ProviderCacheDisablePersistence,
|
||||
},
|
||||
"create_home_cache_config": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.CreateHomeCacheStore,
|
||||
"cache_nodes": cfg.Cache.CreateHomeCacheNodes,
|
||||
"cache_database": cfg.Cache.CreateHomeCacheDatabase,
|
||||
"cache_table": "create_home",
|
||||
"cache_ttl": cfg.Cache.CreateHomeCacheTTL,
|
||||
"cache_size": cfg.Cache.CreateHomeCacheSize,
|
||||
"cache_disable_persistence": cfg.Cache.CreateHomeCacheDisablePersistence,
|
||||
},
|
||||
},
|
||||
"authregistry": map[string]interface{}{
|
||||
"driver": "static",
|
||||
|
||||
@@ -19,24 +19,31 @@ The following image gives an overview of the scenario when a client requests to
|
||||
|
||||
The `graph` service can use a configured store via `GRAPH_CACHE_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `ocmem`: Advanced in-memory store allowing max size.
|
||||
- `redis`: Stores data in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores data in a configured etcd cluster.
|
||||
- `nats-js`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
- `ocmem`: Advanced in-memory store allowing max size. (deprecated)
|
||||
- `redis`: Stores data in a configured Redis cluster. (deprecated)
|
||||
- `etcd`: Stores data in a configured etcd cluster. (deprecated)
|
||||
- `nats-js`: Stores data using object-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/obj_store) (deprecated)
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot-persistent.
|
||||
2. Though usually not necessary, a database name and a database table can be configured for event stores if the event store supports this. Generally not applicable for stores of type `in-memory`. These settings are blank by default which means that the standard settings of the configured store apply.
|
||||
3. The graph service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `GRAPH_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
## Keycloak Configuration For The Personal Data Export
|
||||
|
||||
If Keycloak is used for authentication, GDPR regulations require to add all personal identifiable information that Keycloak has about the user to the personal data export. To do this, the following environment variables must be set:
|
||||
|
||||
* `OCIS_KEYCLOAK_BASE_PATH` - The URL to the keycloak instance.
|
||||
* `OCIS_KEYCLOAK_CLIENT_ID` - The client ID of the client that is used to authenticate with keycloak, this client has to be able to list users and get the credential data.
|
||||
* `OCIS_KEYCLOAK_CLIENT_ID` - The client ID of the client that is used to authenticate with keycloak, this client has to be able to list users and get the credential data.
|
||||
* `OCIS_KEYCLOAK_CLIENT_SECRET` - The client secret of the client that is used to authenticate with keycloak.
|
||||
* `OCIS_KEYCLOAK_CLIENT_REALM` - The realm the client is defined in.
|
||||
* `OCIS_KEYCLOAK_USER_REALM` - The realm the oCIS users are defined in.
|
||||
|
||||
@@ -4,10 +4,11 @@ import "time"
|
||||
|
||||
// Cache defines the available configuration for a cache store
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;GRAPH_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;GRAPH_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"GRAPH_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"GRAPH_CACHE_STORE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;GRAPH_CACHE_TTL" desc:"Time to live for cache records in the graph. Defaults to '336h' (2 weeks). See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;GRAPH_CACHE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;GRAPH_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;GRAPH_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"GRAPH_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"GRAPH_CACHE_STORE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;GRAPH_CACHE_TTL" desc:"Time to live for cache records in the graph. Defaults to '336h' (2 weeks). See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;GRAPH_CACHE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GRAPH_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
}
|
||||
|
||||
@@ -186,6 +186,7 @@ func NewService(opts ...Option) (Graph, error) {
|
||||
microstore.Nodes(options.Config.Cache.Nodes...),
|
||||
microstore.Database(options.Config.Cache.Database),
|
||||
microstore.Table(options.Config.Cache.Table),
|
||||
store.DisablePersistence(options.Config.Cache.DisablePersistence),
|
||||
}
|
||||
m := roles.NewManager(
|
||||
roles.StoreOptions(storeOptions),
|
||||
|
||||
@@ -20,17 +20,24 @@ The `postprocessing` service needs to store some metadata about uploads to be ab
|
||||
|
||||
The `postprocessing` service stores its metadata via the configured store in `POSTPROCESSING_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `ocmem`: Advanced in-memory store allowing max size.
|
||||
- `redis`: Stores data in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores data in a configured etcd cluster.
|
||||
- `nats-js`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
- `ocmem`: Advanced in-memory store allowing max size. (deprecated)
|
||||
- `redis`: Stores data in a configured Redis cluster. (deprecated)
|
||||
- `etcd`: Stores data in a configured etcd cluster. (deprecated)
|
||||
- `nats-js`: Stores data using object-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/obj_store) (deprecated)
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot-persistent.
|
||||
2. Though usually not necessary, a database name and a database table can be configured for event stores if the event store supports this. Generally not applicable for stores of type `in-memory`. These settings are blank by default which means that the standard settings of the configured store apply.
|
||||
3. The postprocessing service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `POSTPROCESSING_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
## Additional Prerequisites for the Postprocessing Service
|
||||
|
||||
|
||||
@@ -134,17 +134,24 @@ In a production deployment, you want to have basic authentication (`PROXY_ENABLE
|
||||
|
||||
The `proxy` service can use a configured store via `PROXY_OIDC_USERINFO_CACHE_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `ocmem`: Advanced in-memory store allowing max size.
|
||||
- `redis`: Stores data in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores data in a configured etcd cluster.
|
||||
- `nats-js`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
- `ocmem`: Advanced in-memory store allowing max size. (deprecated)
|
||||
- `redis`: Stores data in a configured Redis cluster. (deprecated)
|
||||
- `etcd`: Stores data in a configured etcd cluster. (deprecated)
|
||||
- `nats-js`: Stores data using object-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/obj_store) (deprecated)
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot-persistent.
|
||||
2. Though usually not necessary, a database name and a database table can be configured for event stores if the event store supports this. Generally not applicable for stores of type `in-memory`. These settings are blank by default which means that the standard settings of the configured store apply.
|
||||
3. The proxy service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `PROXY_OIDC_USERINFO_CACHE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
## Special Settings
|
||||
|
||||
|
||||
@@ -62,6 +62,7 @@ func Server(cfg *config.Config) *cli.Command {
|
||||
microstore.Nodes(cfg.OIDC.UserinfoCache.Nodes...),
|
||||
microstore.Database(cfg.OIDC.UserinfoCache.Database),
|
||||
microstore.Table(cfg.OIDC.UserinfoCache.Table),
|
||||
store.DisablePersistence(cfg.OIDC.UserinfoCache.DisablePersistence),
|
||||
)
|
||||
|
||||
logger := logging.Configure(cfg.Service.Name, cfg.Log)
|
||||
|
||||
@@ -123,12 +123,13 @@ type JWKS struct {
|
||||
|
||||
// Cache is a TTL cache configuration.
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;PROXY_OIDC_USERINFO_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;PROXY_OIDC_USERINFO_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"PROXY_OIDC_USERINFO_CACHE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;PROXY_OIDC_USERINFO_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;PROXY_OIDC_USERINFO_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;PROXY_OIDC_USERINFO_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;PROXY_OIDC_USERINFO_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"PROXY_OIDC_USERINFO_CACHE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;PROXY_OIDC_USERINFO_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;PROXY_OIDC_USERINFO_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;PROXY_OIDC_USERINFO_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
}
|
||||
|
||||
// RoleAssignment contains the configuration for how to assign roles to users during login
|
||||
|
||||
@@ -41,26 +41,26 @@ graph TD
|
||||
When using `SETTINGS_STORE_TYPE=metadata`, the `settings` service caches the results of queries against the storage backend to provide faster responses. The content of this cache is independent of the cache used in the `storage-system` service as it caches directory listing and settings content stored in files.
|
||||
|
||||
The store used for the cache can be configured using the `SETTINGS_CACHE_STORE` environment variable. Possible stores are:
|
||||
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `redis`: Stores metadata in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores metadata in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores metadata in a configured etcd cluster.
|
||||
- `nats-js`: Stores metadata using the key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
- `ocmem`: Advanced in-memory store allowing max size. (deprecated)
|
||||
- `redis`: Stores data in a configured Redis cluster. (deprecated)
|
||||
- `etcd`: Stores data in a configured etcd cluster. (deprecated)
|
||||
- `nats-js`: Stores data using object-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/obj_store) (deprecated)
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot-persistent.
|
||||
2. Though usually not necessary, a database name can be configured for event
|
||||
stores if the event store supports this. Generally not applicable for
|
||||
stores of type `in-memory`, `redis` and `redis-sentinel`. These settings
|
||||
are blank by default which means that the standard settings of the
|
||||
configured store apply.
|
||||
3. The `settings` service can be scaled if not using `in-memory` stores and
|
||||
the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via
|
||||
`SETTINGS_CACHE_STORE_NODES` in the form of
|
||||
`<sentinel-host>:<sentinel-port>/<redis-master>` like
|
||||
`10.10.0.200:26379/mymaster`.
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
## Settings Management
|
||||
|
||||
|
||||
@@ -57,11 +57,12 @@ type Metadata struct {
|
||||
|
||||
// Cache configures the cache of the Metadata store
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;SETTINGS_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;SETTINGS_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
FileTable string `yaml:"files_table" env:"SETTINGS_FILE_CACHE_TABLE" desc:"The database table the store should use for the file cache."`
|
||||
DirectoryTable string `yaml:"directories_table" env:"SETTINGS_DIRECTORY_CACHE_TABLE" desc:"The database table the store should use for the directory cache."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;SETTINGS_CACHE_TTL" desc:"Default time to live for entries in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;SETTINGS_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;SETTINGS_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;SETTINGS_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
FileTable string `yaml:"files_table" env:"SETTINGS_FILE_CACHE_TABLE" desc:"The database table the store should use for the file cache."`
|
||||
DirectoryTable string `yaml:"directories_table" env:"SETTINGS_DIRECTORY_CACHE_TABLE" desc:"The database table the store should use for the directory cache."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;SETTINGS_CACHE_TTL" desc:"Default time to live for entries in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;SETTINGS_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;SETTINGS_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
}
|
||||
|
||||
@@ -139,6 +139,7 @@ func (c *CachedMDC) Init(ctx context.Context, id string) error {
|
||||
microstore.Nodes(c.cfg.Metadata.Cache.Nodes...),
|
||||
microstore.Database(c.cfg.Metadata.Cache.Database),
|
||||
microstore.Table(c.cfg.Metadata.Cache.DirectoryTable),
|
||||
store.DisablePersistence(c.cfg.Metadata.Cache.DisablePersistence),
|
||||
)
|
||||
c.filesCache = store.Create(
|
||||
store.Store(c.cfg.Metadata.Cache.Store),
|
||||
@@ -147,6 +148,7 @@ func (c *CachedMDC) Init(ctx context.Context, id string) error {
|
||||
microstore.Nodes(c.cfg.Metadata.Cache.Nodes...),
|
||||
microstore.Database(c.cfg.Metadata.Cache.Database),
|
||||
microstore.Table(c.cfg.Metadata.Cache.FileTable),
|
||||
store.DisablePersistence(c.cfg.Metadata.Cache.DisablePersistence),
|
||||
)
|
||||
return c.next.Init(ctx, id)
|
||||
}
|
||||
|
||||
@@ -10,13 +10,21 @@ Starting with ocis version 3.0.0, the default backend for metadata switched to m
|
||||
|
||||
The `storage-system` service caches file metadata via the configured store in `STORAGE_SYSTEM_CACHE_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `redis`: Stores metadata in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores metadata in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores metadata in a configured etcd cluster.
|
||||
- `nats-js`: Stores metadata using the key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
- `ocmem`: Advanced in-memory store allowing max size. (deprecated)
|
||||
- `redis`: Stores data in a configured Redis cluster. (deprecated)
|
||||
- `etcd`: Stores data in a configured etcd cluster. (deprecated)
|
||||
- `nats-js`: Stores data using object-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/obj_store) (deprecated)
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot-persistent.
|
||||
2. Though usually not necessary, a database name can be configured for event stores if the event store supports this. Generally not applicable for stores of type `in-memory`, `redis` and `redis-sentinel`. These settings are blank by default which means that the standard settings of the configured store apply.
|
||||
3. The `storage-system` service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `STORAGE_SYSTEM_CACHE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
@@ -87,9 +87,10 @@ type OCISDriver struct {
|
||||
|
||||
// Cache holds cache config
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_SYSTEM_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_SYSTEM_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_SYSTEM_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_SYSTEM_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_SYSTEM_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_SYSTEM_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_SYSTEM_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_SYSTEM_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_SYSTEM_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
}
|
||||
|
||||
@@ -162,11 +162,12 @@ func metadataDrivers(cfg *config.Config) map[string]interface{} {
|
||||
"cache_database": "system",
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.FileMetadataCache.Store,
|
||||
"cache_nodes": cfg.FileMetadataCache.Nodes,
|
||||
"cache_database": cfg.FileMetadataCache.Database,
|
||||
"cache_ttl": cfg.FileMetadataCache.TTL / time.Second,
|
||||
"cache_size": cfg.FileMetadataCache.Size,
|
||||
"cache_store": cfg.FileMetadataCache.Store,
|
||||
"cache_nodes": cfg.FileMetadataCache.Nodes,
|
||||
"cache_database": cfg.FileMetadataCache.Database,
|
||||
"cache_ttl": cfg.FileMetadataCache.TTL / time.Second,
|
||||
"cache_size": cfg.FileMetadataCache.Size,
|
||||
"cache_disable_persistence": cfg.FileMetadataCache.DisablePersistence,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -10,13 +10,13 @@ Starting with ocis version 3.0.0, the default backend for metadata switched to m
|
||||
|
||||
Starting with Infinite Scale version 3.1, you can define a graceful shutdown period for the `storage-users` service.
|
||||
|
||||
IMPORTANT: The graceful shutdown period is only applicable if the `storage-users` service runs as standalone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a standalone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that both instances must be able to communicate on the same network.
|
||||
IMPORTANT: The graceful shutdown period is only applicable if the `storage-users` service runs as standalone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a standalone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that both instances must be able to communicate on the same network.
|
||||
|
||||
When hard-stopping Infinite Scale, for example with the `kill <pid>` command (SIGKILL), it is possible and likely that not all data from the decomposedfs (metadata) has been written to the storage which may result in an inconsistent decomposedfs. When gracefully shutting down Infinite Scale, using a command like SIGTERM, the process will no longer accept any write requests from _other_ services and will try to write the internal open requests which can take an undefined duration based on many factors. To mitigate that situation, the following things have been implemented:
|
||||
|
||||
* With the value of the environment variable `STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT`, the `storage-users` service will delay its shutdown giving it time to finalize writing necessary data. This delay can be necessary if there is a lot of data to be saved and/or if storage access/thruput is slow. In such a case you would receive an error log entry informing you that not all data could be saved in time. To prevent such occurrences, you must increase the default value.
|
||||
|
||||
* If a shutdown error has been logged, the command-line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details.
|
||||
* If a shutdown error has been logged, the command-line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details.
|
||||
|
||||
## CLI Commands
|
||||
|
||||
@@ -37,7 +37,7 @@ When using Infinite Scale as user storage, a directory named `storage/users/uplo
|
||||
Example cases for expired uploads
|
||||
|
||||
* When a user uploads a big file but the file exceeds the user-quota, the upload can't be moved to the target after it has finished. The file stays at the upload location until it is manually cleared.
|
||||
* If the bandwidth is limited and the file to transfer can't be transferred completely before the upload expiration time is reached, the file expires and can't be processed.
|
||||
* If the bandwidth is limited and the file to transfer can't be transferred completely before the upload expiration time is reached, the file expires and can't be processed.
|
||||
|
||||
There are two commands available to manage unfinished uploads
|
||||
|
||||
@@ -101,13 +101,21 @@ The configuration for the `purge-expired` command is done by using the following
|
||||
|
||||
The `storage-users` service caches stat, metadata and uuids of files and folders via the configured store in `STORAGE_USERS_STAT_CACHE_STORE`, `STORAGE_USERS_FILEMETADATA_CACHE_STORE` and `STORAGE_USERS_ID_CACHE_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `redis`: Stores metadata in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores metadata in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores metadata in a configured etcd cluster.
|
||||
- `nats-js`: Stores metadata using the key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
- `ocmem`: Advanced in-memory store allowing max size. (deprecated)
|
||||
- `redis`: Stores data in a configured Redis cluster. (deprecated)
|
||||
- `etcd`: Stores data in a configured etcd cluster. (deprecated)
|
||||
- `nats-js`: Stores data using object-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/obj_store) (deprecated)
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot-persistent.
|
||||
2. Though usually not necessary, a database name can be configured for event stores if the event store supports this. Generally not applicable for stores of type `in-memory`, `redis` and `redis-sentinel`. These settings are blank by default which means that the standard settings of the configured store apply.
|
||||
3. The `storage-users` service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `STORAGE_USERS_STAT_CACHE_STORE_NODES`, `STORAGE_USERS_FILEMETADATA_CACHE_STORE_NODES` and `STORAGE_USERS_ID_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
@@ -176,29 +176,32 @@ type Events struct {
|
||||
|
||||
// StatCache holds cache config
|
||||
type StatCache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_STAT_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_STAT_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_USERS_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
}
|
||||
|
||||
// FilemetadataCache holds cache config
|
||||
type FilemetadataCache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_FILEMETADATA_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_FILEMETADATA_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_FILEMETADATA_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_FILEMETADATA_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_FILEMETADATA_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_FILEMETADATA_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_FILEMETADATA_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_FILEMETADATA_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_USERS_FILEMETADATA_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
}
|
||||
|
||||
// IDCache holds cache config
|
||||
type IDCache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_ID_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_ID_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_ID_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens have no expiration. Defaults to 300s which is derived from the underlaying package though not explicitely set as default. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_ID_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_ID_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_ID_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_ID_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens have no expiration. Defaults to 300s which is derived from the underlaying package though not explicitly set as default. See the Environment Variable Types description for more details."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_ID_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_USERS_ID_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
|
||||
}
|
||||
|
||||
// S3Driver is the storage driver configuration when using 's3' storage driver
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
package revaconfig
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/storage-users/pkg/config"
|
||||
)
|
||||
|
||||
@@ -74,28 +72,31 @@ func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} {
|
||||
"nats_enable_tls": cfg.Events.EnableTLS,
|
||||
"data_txs": map[string]interface{}{
|
||||
"simple": map[string]interface{}{
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL / time.Second,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_table": "stat",
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_table": "stat",
|
||||
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
|
||||
},
|
||||
"spaces": map[string]interface{}{
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL / time.Second,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_table": "stat",
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_table": "stat",
|
||||
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
|
||||
},
|
||||
"tus": map[string]interface{}{
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL / time.Second,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_table": "stat",
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_table": "stat",
|
||||
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package revaconfig
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/storage-users/pkg/config"
|
||||
)
|
||||
|
||||
@@ -135,25 +133,28 @@ func Ocis(cfg *config.Config) map[string]interface{} {
|
||||
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
|
||||
"max_quota": cfg.Drivers.OCIS.MaxQuota,
|
||||
"statcache": map[string]interface{}{
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL / time.Second,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.FilemetadataCache.Store,
|
||||
"cache_nodes": cfg.FilemetadataCache.Nodes,
|
||||
"cache_database": cfg.FilemetadataCache.Database,
|
||||
"cache_ttl": cfg.FilemetadataCache.TTL / time.Second,
|
||||
"cache_size": cfg.FilemetadataCache.Size,
|
||||
"cache_store": cfg.FilemetadataCache.Store,
|
||||
"cache_nodes": cfg.FilemetadataCache.Nodes,
|
||||
"cache_database": cfg.FilemetadataCache.Database,
|
||||
"cache_ttl": cfg.FilemetadataCache.TTL,
|
||||
"cache_size": cfg.FilemetadataCache.Size,
|
||||
"cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence,
|
||||
},
|
||||
"idcache": map[string]interface{}{
|
||||
"cache_store": cfg.IDCache.Store,
|
||||
"cache_nodes": cfg.IDCache.Nodes,
|
||||
"cache_database": cfg.IDCache.Database,
|
||||
"cache_ttl": cfg.IDCache.TTL / time.Second,
|
||||
"cache_size": cfg.IDCache.Size,
|
||||
"cache_store": cfg.IDCache.Store,
|
||||
"cache_nodes": cfg.IDCache.Nodes,
|
||||
"cache_database": cfg.IDCache.Database,
|
||||
"cache_ttl": cfg.IDCache.TTL,
|
||||
"cache_size": cfg.IDCache.Size,
|
||||
"cache_disable_persistence": cfg.IDCache.DisablePersistence,
|
||||
},
|
||||
"events": map[string]interface{}{
|
||||
"natsaddress": cfg.Events.Addr,
|
||||
@@ -193,25 +194,28 @@ func OcisNoEvents(cfg *config.Config) map[string]interface{} {
|
||||
"max_concurrency": cfg.Drivers.OCIS.MaxConcurrency,
|
||||
"max_quota": cfg.Drivers.OCIS.MaxQuota,
|
||||
"statcache": map[string]interface{}{
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL / time.Second,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.FilemetadataCache.Store,
|
||||
"cache_nodes": cfg.FilemetadataCache.Nodes,
|
||||
"cache_database": cfg.FilemetadataCache.Database,
|
||||
"cache_ttl": cfg.FilemetadataCache.TTL / time.Second,
|
||||
"cache_size": cfg.FilemetadataCache.Size,
|
||||
"cache_store": cfg.FilemetadataCache.Store,
|
||||
"cache_nodes": cfg.FilemetadataCache.Nodes,
|
||||
"cache_database": cfg.FilemetadataCache.Database,
|
||||
"cache_ttl": cfg.FilemetadataCache.TTL,
|
||||
"cache_size": cfg.FilemetadataCache.Size,
|
||||
"cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence,
|
||||
},
|
||||
"idcache": map[string]interface{}{
|
||||
"cache_store": cfg.IDCache.Store,
|
||||
"cache_nodes": cfg.IDCache.Nodes,
|
||||
"cache_database": cfg.IDCache.Database,
|
||||
"cache_ttl": cfg.IDCache.TTL / time.Second,
|
||||
"cache_size": cfg.IDCache.Size,
|
||||
"cache_store": cfg.IDCache.Store,
|
||||
"cache_nodes": cfg.IDCache.Nodes,
|
||||
"cache_database": cfg.IDCache.Database,
|
||||
"cache_ttl": cfg.IDCache.TTL,
|
||||
"cache_size": cfg.IDCache.Size,
|
||||
"cache_disable_persistence": cfg.IDCache.DisablePersistence,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -256,25 +260,28 @@ func S3NG(cfg *config.Config) map[string]interface{} {
|
||||
"max_concurrency": cfg.Drivers.S3NG.MaxConcurrency,
|
||||
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
|
||||
"statcache": map[string]interface{}{
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL / time.Second,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.FilemetadataCache.Store,
|
||||
"cache_nodes": cfg.FilemetadataCache.Nodes,
|
||||
"cache_database": cfg.FilemetadataCache.Database,
|
||||
"cache_ttl": cfg.FilemetadataCache.TTL / time.Second,
|
||||
"cache_size": cfg.FilemetadataCache.Size,
|
||||
"cache_store": cfg.FilemetadataCache.Store,
|
||||
"cache_nodes": cfg.FilemetadataCache.Nodes,
|
||||
"cache_database": cfg.FilemetadataCache.Database,
|
||||
"cache_ttl": cfg.FilemetadataCache.TTL,
|
||||
"cache_size": cfg.FilemetadataCache.Size,
|
||||
"cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence,
|
||||
},
|
||||
"idcache": map[string]interface{}{
|
||||
"cache_store": cfg.IDCache.Store,
|
||||
"cache_nodes": cfg.IDCache.Nodes,
|
||||
"cache_database": cfg.IDCache.Database,
|
||||
"cache_ttl": cfg.IDCache.TTL / time.Second,
|
||||
"cache_size": cfg.IDCache.Size,
|
||||
"cache_store": cfg.IDCache.Store,
|
||||
"cache_nodes": cfg.IDCache.Nodes,
|
||||
"cache_database": cfg.IDCache.Database,
|
||||
"cache_ttl": cfg.IDCache.TTL,
|
||||
"cache_size": cfg.IDCache.Size,
|
||||
"cache_disable_persistence": cfg.IDCache.DisablePersistence,
|
||||
},
|
||||
"events": map[string]interface{}{
|
||||
"natsaddress": cfg.Events.Addr,
|
||||
@@ -318,25 +325,28 @@ func S3NGNoEvents(cfg *config.Config) map[string]interface{} {
|
||||
"max_concurrency": cfg.Drivers.S3NG.MaxConcurrency,
|
||||
"lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor,
|
||||
"statcache": map[string]interface{}{
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL / time.Second,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_store": cfg.StatCache.Store,
|
||||
"cache_nodes": cfg.StatCache.Nodes,
|
||||
"cache_database": cfg.StatCache.Database,
|
||||
"cache_ttl": cfg.StatCache.TTL,
|
||||
"cache_size": cfg.StatCache.Size,
|
||||
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.FilemetadataCache.Store,
|
||||
"cache_nodes": cfg.FilemetadataCache.Nodes,
|
||||
"cache_database": cfg.FilemetadataCache.Database,
|
||||
"cache_ttl": cfg.FilemetadataCache.TTL / time.Second,
|
||||
"cache_size": cfg.FilemetadataCache.Size,
|
||||
"cache_store": cfg.FilemetadataCache.Store,
|
||||
"cache_nodes": cfg.FilemetadataCache.Nodes,
|
||||
"cache_database": cfg.FilemetadataCache.Database,
|
||||
"cache_ttl": cfg.FilemetadataCache.TTL,
|
||||
"cache_size": cfg.FilemetadataCache.Size,
|
||||
"cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence,
|
||||
},
|
||||
"idcache": map[string]interface{}{
|
||||
"cache_store": cfg.IDCache.Store,
|
||||
"cache_nodes": cfg.IDCache.Nodes,
|
||||
"cache_database": cfg.IDCache.Database,
|
||||
"cache_ttl": cfg.IDCache.TTL / time.Second,
|
||||
"cache_size": cfg.IDCache.Size,
|
||||
"cache_store": cfg.IDCache.Store,
|
||||
"cache_nodes": cfg.IDCache.Nodes,
|
||||
"cache_database": cfg.IDCache.Database,
|
||||
"cache_ttl": cfg.IDCache.TTL,
|
||||
"cache_size": cfg.IDCache.Size,
|
||||
"cache_disable_persistence": cfg.IDCache.DisablePersistence,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,17 +17,24 @@ Running the `userlog` service without running the `eventhistory` service is not
|
||||
|
||||
The `userlog` service persists information via the configured store in `USERLOG_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `ocmem`: Advanced in-memory store allowing max size.
|
||||
- `redis`: Stores data in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores data in a configured etcd cluster.
|
||||
- `nats-js`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
- `ocmem`: Advanced in-memory store allowing max size. (deprecated)
|
||||
- `redis`: Stores data in a configured Redis cluster. (deprecated)
|
||||
- `etcd`: Stores data in a configured etcd cluster. (deprecated)
|
||||
- `nats-js`: Stores data using object-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/obj_store) (deprecated)
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot-persistent.
|
||||
2. Though usually not necessary, a database name and a database table can be configured for event stores if the event store supports this. Generally not applicable for stores of type `in-memory`. These settings are blank by default which means that the standard settings of the configured store apply.
|
||||
3. The userlog service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `USERLOG_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
## Configuring
|
||||
|
||||
|
||||
65
vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/gateway.go
generated
vendored
65
vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/gateway.go
generated
vendored
@@ -22,7 +22,6 @@ import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/cs3org/reva/v2/pkg/errtypes"
|
||||
@@ -64,26 +63,14 @@ type config struct {
|
||||
TokenManager string `mapstructure:"token_manager"`
|
||||
// ShareFolder is the location where to create shares in the recipient's storage provider.
|
||||
// FIXME get rid of ShareFolder, there are findByPath calls in the ocmshareporvider.go and usershareprovider.go
|
||||
ShareFolder string `mapstructure:"share_folder"`
|
||||
DataTransfersFolder string `mapstructure:"data_transfers_folder"`
|
||||
TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"`
|
||||
AllowedUserAgents map[string][]string `mapstructure:"allowed_user_agents"` // map[path][]user-agent
|
||||
StatCacheStore string `mapstructure:"stat_cache_store"`
|
||||
StatCacheNodes []string `mapstructure:"stat_cache_nodes"`
|
||||
StatCacheDatabase string `mapstructure:"stat_cache_database"`
|
||||
StatCacheTTL int `mapstructure:"stat_cache_ttl"`
|
||||
StatCacheSize int `mapstructure:"stat_cache_size"`
|
||||
CreateHomeCacheStore string `mapstructure:"create_home_cache_store"`
|
||||
CreateHomeCacheNodes []string `mapstructure:"create_home_cache_nodes"`
|
||||
CreateHomeCacheDatabase string `mapstructure:"create_home_cache_database"`
|
||||
CreateHomeCacheTTL int `mapstructure:"create_home_cache_ttl"`
|
||||
CreateHomeCacheSize int `mapstructure:"create_home_cache_size"`
|
||||
ProviderCacheStore string `mapstructure:"provider_cache_store"`
|
||||
ProviderCacheNodes []string `mapstructure:"provider_cache_nodes"`
|
||||
ProviderCacheDatabase string `mapstructure:"provider_cache_database"`
|
||||
ProviderCacheTTL int `mapstructure:"provider_cache_ttl"`
|
||||
ProviderCacheSize int `mapstructure:"provider_cache_size"`
|
||||
UseCommonSpaceRootShareLogic bool `mapstructure:"use_common_space_root_share_logic"`
|
||||
ShareFolder string `mapstructure:"share_folder"`
|
||||
DataTransfersFolder string `mapstructure:"data_transfers_folder"`
|
||||
TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"`
|
||||
AllowedUserAgents map[string][]string `mapstructure:"allowed_user_agents"` // map[path][]user-agent
|
||||
StatCacheConfig cache.Config `mapstructure:"stat_cache_config"`
|
||||
CreatePersonalSpaceCacheConfig cache.Config `mapstructure:"create_personal_space_cache_config"`
|
||||
ProviderCacheConfig cache.Config `mapstructure:"provider_cache_config"`
|
||||
UseCommonSpaceRootShareLogic bool `mapstructure:"use_common_space_root_share_logic"`
|
||||
}
|
||||
|
||||
// sets defaults
|
||||
@@ -130,28 +117,28 @@ func (c *config) init() {
|
||||
}
|
||||
|
||||
// caching needs to be explicitly enabled
|
||||
if c.StatCacheStore == "" {
|
||||
c.StatCacheStore = "noop"
|
||||
if c.StatCacheConfig.Store == "" {
|
||||
c.StatCacheConfig.Store = "noop"
|
||||
}
|
||||
|
||||
if c.StatCacheDatabase == "" {
|
||||
c.StatCacheDatabase = "reva"
|
||||
if c.StatCacheConfig.Database == "" {
|
||||
c.StatCacheConfig.Database = "reva"
|
||||
}
|
||||
|
||||
if c.ProviderCacheStore == "" {
|
||||
c.ProviderCacheStore = "noop"
|
||||
if c.ProviderCacheConfig.Store == "" {
|
||||
c.ProviderCacheConfig.Store = "noop"
|
||||
}
|
||||
|
||||
if c.ProviderCacheDatabase == "" {
|
||||
c.ProviderCacheDatabase = "reva"
|
||||
if c.ProviderCacheConfig.Database == "" {
|
||||
c.ProviderCacheConfig.Database = "reva"
|
||||
}
|
||||
|
||||
if c.CreateHomeCacheStore == "" {
|
||||
c.CreateHomeCacheStore = "noop"
|
||||
if c.CreatePersonalSpaceCacheConfig.Store == "" {
|
||||
c.CreatePersonalSpaceCacheConfig.Store = "noop"
|
||||
}
|
||||
|
||||
if c.CreateHomeCacheDatabase == "" {
|
||||
c.CreateHomeCacheDatabase = "reva"
|
||||
if c.CreatePersonalSpaceCacheConfig.Database == "" {
|
||||
c.CreatePersonalSpaceCacheConfig.Database = "reva"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,14 +148,13 @@ type svc struct {
|
||||
tokenmgr token.Manager
|
||||
statCache cache.StatCache
|
||||
providerCache cache.ProviderCache
|
||||
createHomeCache cache.CreateHomeCache
|
||||
createPersonalSpaceCache cache.CreatePersonalSpaceCache
|
||||
}
|
||||
|
||||
// New creates a new gateway svc that acts as a proxy for any grpc operation.
|
||||
// The gateway is responsible for high-level controls: rate-limiting, coordination between svcs
|
||||
// like sharing and storage acls, asynchronous transactions, ...
|
||||
func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
|
||||
func New(m map[string]interface{}, _ *grpc.Server) (rgrpc.Service, error) {
|
||||
c, err := parseConfig(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -191,10 +177,9 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
|
||||
c: c,
|
||||
dataGatewayURL: *u,
|
||||
tokenmgr: tokenManager,
|
||||
statCache: cache.GetStatCache(c.StatCacheStore, c.StatCacheNodes, c.StatCacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize),
|
||||
providerCache: cache.GetProviderCache(c.ProviderCacheStore, c.ProviderCacheNodes, c.ProviderCacheDatabase, "provider", time.Duration(c.ProviderCacheTTL)*time.Second, c.ProviderCacheSize),
|
||||
createHomeCache: cache.GetCreateHomeCache(c.CreateHomeCacheStore, c.CreateHomeCacheNodes, c.CreateHomeCacheDatabase, "createHome", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize),
|
||||
createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CreateHomeCacheStore, c.CreateHomeCacheNodes, c.CreateHomeCacheDatabase, "createPersonalSpace", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize),
|
||||
statCache: cache.GetStatCache(c.StatCacheConfig),
|
||||
providerCache: cache.GetProviderCache(c.ProviderCacheConfig),
|
||||
createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CreatePersonalSpaceCacheConfig),
|
||||
}
|
||||
|
||||
return s, nil
|
||||
@@ -207,7 +192,7 @@ func (s *svc) Register(ss *grpc.Server) {
|
||||
func (s *svc) Close() error {
|
||||
s.statCache.Close()
|
||||
s.providerCache.Close()
|
||||
s.createHomeCache.Close()
|
||||
s.createPersonalSpaceCache.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
1
vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/storageprovider.go
generated
vendored
1
vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/storageprovider.go
generated
vendored
@@ -1107,7 +1107,6 @@ func (s *svc) getStorageProviderClient(_ context.Context, p *registry.ProviderIn
|
||||
return &cachedAPIClient{
|
||||
c: c,
|
||||
statCache: s.statCache,
|
||||
createHomeCache: s.createHomeCache,
|
||||
createPersonalSpaceCache: s.createPersonalSpaceCache,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -85,7 +85,6 @@ func (c *cachedRegistryClient) GetHome(ctx context.Context, in *registry.GetHome
|
||||
type cachedAPIClient struct {
|
||||
c provider.ProviderAPIClient
|
||||
statCache cache.StatCache
|
||||
createHomeCache cache.CreateHomeCache
|
||||
createPersonalSpaceCache cache.CreatePersonalSpaceCache
|
||||
}
|
||||
|
||||
@@ -121,10 +120,10 @@ func (c *cachedAPIClient) Stat(ctx context.Context, in *provider.StatRequest, op
|
||||
|
||||
// CreateHome caches calls to CreateHome locally - anyways they only need to be called once per user
|
||||
func (c *cachedAPIClient) CreateHome(ctx context.Context, in *provider.CreateHomeRequest, opts ...grpc.CallOption) (*provider.CreateHomeResponse, error) {
|
||||
key := c.createHomeCache.GetKey(ctxpkg.ContextMustGetUser(ctx).GetId())
|
||||
key := c.createPersonalSpaceCache.GetKey(ctxpkg.ContextMustGetUser(ctx).GetId())
|
||||
if key != "" {
|
||||
s := &provider.CreateHomeResponse{}
|
||||
if err := c.createHomeCache.PullFromCache(key, s); err == nil {
|
||||
if err := c.createPersonalSpaceCache.PullFromCache(key, s); err == nil {
|
||||
return s, nil
|
||||
}
|
||||
}
|
||||
@@ -137,7 +136,7 @@ func (c *cachedAPIClient) CreateHome(ctx context.Context, in *provider.CreateHom
|
||||
case key == "":
|
||||
return resp, nil
|
||||
default:
|
||||
return resp, c.createHomeCache.PushToCache(key, resp)
|
||||
return resp, c.createPersonalSpaceCache.PushToCache(key, resp)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ package config
|
||||
import (
|
||||
"github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data"
|
||||
"github.com/cs3org/reva/v2/pkg/sharedconf"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/cache"
|
||||
)
|
||||
|
||||
// Config holds the config options that need to be passed down to all ocs handlers
|
||||
@@ -37,12 +38,7 @@ type Config struct {
|
||||
AdditionalInfoAttribute string `mapstructure:"additional_info_attribute"`
|
||||
CacheWarmupDriver string `mapstructure:"cache_warmup_driver"`
|
||||
CacheWarmupDrivers map[string]map[string]interface{} `mapstructure:"cache_warmup_drivers"`
|
||||
StatCacheStore string `mapstructure:"stat_cache_store"`
|
||||
StatCacheNodes []string `mapstructure:"stat_cache_nodes"`
|
||||
StatCacheDatabase string `mapstructure:"stat_cache_database"`
|
||||
StatCacheTable string `mapstructure:"stat_cache_table"`
|
||||
StatCacheTTL int `mapstructure:"stat_cache_ttl"`
|
||||
StatCacheSize int `mapstructure:"stat_cache_size"`
|
||||
StatCacheConfig cache.Config `mapstructure:"stat_cache_config"`
|
||||
UserIdentifierCacheTTL int `mapstructure:"user_identifier_cache_ttl"`
|
||||
MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"`
|
||||
SkipUpdatingExistingSharesMountpoints bool `mapstructure:"skip_updating_existing_shares_mountpoint"`
|
||||
|
||||
@@ -150,7 +150,7 @@ func (h *Handler) Init(c *config.Config) error {
|
||||
h.publicPasswordEnforced = publicPwdEnforced(c)
|
||||
h.passwordValidator = passwordPolicies(c)
|
||||
|
||||
h.statCache = cache.GetStatCache(c.StatCacheStore, c.StatCacheNodes, c.StatCacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize)
|
||||
h.statCache = cache.GetStatCache(c.StatCacheConfig)
|
||||
if c.CacheWarmupDriver != "" {
|
||||
cwm, err := getCacheWarmupManager(c)
|
||||
if err != nil {
|
||||
|
||||
5
vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/ocs.go
generated
vendored
5
vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/ocs.go
generated
vendored
@@ -20,7 +20,6 @@ package ocs
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/config"
|
||||
"github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/sharees"
|
||||
@@ -67,9 +66,9 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if conf.CacheWarmupDriver == "first-request" && conf.StatCacheStore != "noop" {
|
||||
if conf.CacheWarmupDriver == "first-request" && conf.StatCacheConfig.Store != "noop" {
|
||||
s.warmupCacheTracker = ttlcache.NewCache()
|
||||
_ = s.warmupCacheTracker.SetTTL(time.Second * time.Duration(conf.StatCacheTTL))
|
||||
_ = s.warmupCacheTracker.SetTTL(conf.StatCacheConfig.TTL)
|
||||
}
|
||||
|
||||
return s, nil
|
||||
|
||||
2
vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go
generated
vendored
2
vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go
generated
vendored
@@ -70,7 +70,7 @@ func New(m map[string]interface{}, publisher events.Publisher) (datatx.DataTX, e
|
||||
return &manager{
|
||||
conf: c,
|
||||
publisher: publisher,
|
||||
statCache: cache.GetStatCache(c.Store, c.Nodes, c.Database, c.Table, time.Duration(c.TTL)*time.Second, c.Size),
|
||||
statCache: cache.GetStatCache(*c),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
2
vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/spaces/spaces.go
generated
vendored
2
vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/spaces/spaces.go
generated
vendored
@@ -72,7 +72,7 @@ func New(m map[string]interface{}, publisher events.Publisher) (datatx.DataTX, e
|
||||
return &manager{
|
||||
conf: c,
|
||||
publisher: publisher,
|
||||
statCache: cache.GetStatCache(c.Store, c.Nodes, c.Database, c.Table, time.Duration(c.TTL)*time.Second, c.Size),
|
||||
statCache: cache.GetStatCache(*c),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
3
vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go
generated
vendored
3
vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go
generated
vendored
@@ -23,7 +23,6 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
tusd "github.com/tus/tusd/pkg/handler"
|
||||
@@ -71,7 +70,7 @@ func New(m map[string]interface{}, publisher events.Publisher) (datatx.DataTX, e
|
||||
return &manager{
|
||||
conf: c,
|
||||
publisher: publisher,
|
||||
statCache: cache.GetStatCache(c.Store, c.Nodes, c.Database, c.Table, time.Duration(c.TTL)*time.Second, c.Size),
|
||||
statCache: cache.GetStatCache(*c),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
58
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/cache.go
generated
vendored
58
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/cache.go
generated
vendored
@@ -44,12 +44,13 @@ var (
|
||||
|
||||
// Config contains the configuring for a cache
|
||||
type Config struct {
|
||||
Store string `mapstructure:"cache_store"`
|
||||
Nodes []string `mapstructure:"cache_nodes"`
|
||||
Database string `mapstructure:"cache_database"`
|
||||
Table string `mapstructure:"cache_table"`
|
||||
TTL int `mapstructure:"cache_ttl"`
|
||||
Size int `mapstructure:"cache_size"`
|
||||
Store string `mapstructure:"cache_store"`
|
||||
Nodes []string `mapstructure:"cache_nodes"`
|
||||
Database string `mapstructure:"cache_database"`
|
||||
Table string `mapstructure:"cache_table"`
|
||||
TTL time.Duration `mapstructure:"cache_ttl"`
|
||||
Size int `mapstructure:"cache_size"`
|
||||
DisablePersistence bool `mapstructure:"cache_disable_persistence"`
|
||||
}
|
||||
|
||||
// Cache handles key value operations on caches
|
||||
@@ -100,65 +101,65 @@ type FileMetadataCache interface {
|
||||
|
||||
// GetStatCache will return an existing StatCache for the given store, nodes, database and table
|
||||
// If it does not exist yet it will be created, different TTLs are ignored
|
||||
func GetStatCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration, size int) StatCache {
|
||||
func GetStatCache(cfg Config) StatCache {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
key := strings.Join(append(append([]string{cacheStore}, cacheNodes...), database, table), ":")
|
||||
key := strings.Join(append(append([]string{cfg.Store}, cfg.Nodes...), cfg.Database, cfg.Table), ":")
|
||||
if statCaches[key] == nil {
|
||||
statCaches[key] = NewStatCache(cacheStore, cacheNodes, database, table, ttl, size)
|
||||
statCaches[key] = NewStatCache(cfg)
|
||||
}
|
||||
return statCaches[key]
|
||||
}
|
||||
|
||||
// GetProviderCache will return an existing ProviderCache for the given store, nodes, database and table
|
||||
// If it does not exist yet it will be created, different TTLs are ignored
|
||||
func GetProviderCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration, size int) ProviderCache {
|
||||
func GetProviderCache(cfg Config) ProviderCache {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
key := strings.Join(append(append([]string{cacheStore}, cacheNodes...), database, table), ":")
|
||||
key := strings.Join(append(append([]string{cfg.Store}, cfg.Nodes...), cfg.Database, cfg.Table), ":")
|
||||
if providerCaches[key] == nil {
|
||||
providerCaches[key] = NewProviderCache(cacheStore, cacheNodes, database, table, ttl, size)
|
||||
providerCaches[key] = NewProviderCache(cfg)
|
||||
}
|
||||
return providerCaches[key]
|
||||
}
|
||||
|
||||
// GetCreateHomeCache will return an existing CreateHomeCache for the given store, nodes, database and table
|
||||
// If it does not exist yet it will be created, different TTLs are ignored
|
||||
func GetCreateHomeCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration, size int) CreateHomeCache {
|
||||
func GetCreateHomeCache(cfg Config) CreateHomeCache {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
key := strings.Join(append(append([]string{cacheStore}, cacheNodes...), database, table), ":")
|
||||
key := strings.Join(append(append([]string{cfg.Store}, cfg.Nodes...), cfg.Database, cfg.Table), ":")
|
||||
if createHomeCaches[key] == nil {
|
||||
createHomeCaches[key] = NewCreateHomeCache(cacheStore, cacheNodes, database, table, ttl, size)
|
||||
createHomeCaches[key] = NewCreateHomeCache(cfg)
|
||||
}
|
||||
return createHomeCaches[key]
|
||||
}
|
||||
|
||||
// GetCreatePersonalSpaceCache will return an existing CreatePersonalSpaceCache for the given store, nodes, database and table
|
||||
// If it does not exist yet it will be created, different TTLs are ignored
|
||||
func GetCreatePersonalSpaceCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration, size int) CreatePersonalSpaceCache {
|
||||
func GetCreatePersonalSpaceCache(cfg Config) CreatePersonalSpaceCache {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
key := strings.Join(append(append([]string{cacheStore}, cacheNodes...), database, table), ":")
|
||||
key := strings.Join(append(append([]string{cfg.Store}, cfg.Nodes...), cfg.Database, cfg.Table), ":")
|
||||
if createPersonalSpaceCaches[key] == nil {
|
||||
createPersonalSpaceCaches[key] = NewCreatePersonalSpaceCache(cacheStore, cacheNodes, database, table, ttl, size)
|
||||
createPersonalSpaceCaches[key] = NewCreatePersonalSpaceCache(cfg)
|
||||
}
|
||||
return createPersonalSpaceCaches[key]
|
||||
}
|
||||
|
||||
// GetFileMetadataCache will return an existing GetFileMetadataCache for the given store, nodes, database and table
|
||||
// If it does not exist yet it will be created, different TTLs are ignored
|
||||
func GetFileMetadataCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration, size int) FileMetadataCache {
|
||||
func GetFileMetadataCache(cfg Config) FileMetadataCache {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
key := strings.Join(append(append([]string{cacheStore}, cacheNodes...), database, table), ":")
|
||||
key := strings.Join(append(append([]string{cfg.Store}, cfg.Nodes...), cfg.Database, cfg.Table), ":")
|
||||
if fileMetadataCaches[key] == nil {
|
||||
fileMetadataCaches[key] = NewFileMetadataCache(cacheStore, cacheNodes, database, table, ttl, size)
|
||||
fileMetadataCaches[key] = NewFileMetadataCache(cfg)
|
||||
}
|
||||
return fileMetadataCaches[key]
|
||||
}
|
||||
@@ -230,13 +231,14 @@ func (cache cacheStore) Close() error {
|
||||
return cache.s.Close()
|
||||
}
|
||||
|
||||
func getStore(storeType string, nodes []string, database, table string, ttl time.Duration, size int) microstore.Store {
|
||||
func getStore(cfg Config) microstore.Store {
|
||||
return store.Create(
|
||||
store.Store(storeType),
|
||||
microstore.Nodes(nodes...),
|
||||
microstore.Database(database),
|
||||
microstore.Table(table),
|
||||
store.TTL(ttl),
|
||||
store.Size(size),
|
||||
store.Store(cfg.Store),
|
||||
microstore.Nodes(cfg.Nodes...),
|
||||
microstore.Database(cfg.Database),
|
||||
microstore.Table(cfg.Table),
|
||||
store.TTL(cfg.TTL),
|
||||
store.Size(cfg.Size),
|
||||
store.DisablePersistence(cfg.DisablePersistence),
|
||||
)
|
||||
}
|
||||
|
||||
11
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/createhome.go
generated
vendored
11
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/createhome.go
generated
vendored
@@ -20,7 +20,6 @@ package cache
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
@@ -32,12 +31,12 @@ type createHomeCache struct {
|
||||
}
|
||||
|
||||
// NewCreateHomeCache creates a new CreateHomeCache
|
||||
func NewCreateHomeCache(store string, nodes []string, database, table string, ttl time.Duration, size int) CreateHomeCache {
|
||||
func NewCreateHomeCache(cfg Config) CreateHomeCache {
|
||||
c := &createHomeCache{}
|
||||
c.s = getStore(store, nodes, database, table, ttl, size)
|
||||
c.database = database
|
||||
c.table = table
|
||||
c.ttl = ttl
|
||||
c.s = getStore(cfg)
|
||||
c.database = cfg.Database
|
||||
c.table = cfg.Table
|
||||
c.ttl = cfg.TTL
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
12
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/createpersonalspace.go
generated
vendored
12
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/createpersonalspace.go
generated
vendored
@@ -19,8 +19,6 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
)
|
||||
|
||||
@@ -30,12 +28,12 @@ type createPersonalSpaceCache struct {
|
||||
}
|
||||
|
||||
// NewCreatePersonalSpaceCache creates a new CreatePersonalSpaceCache
|
||||
func NewCreatePersonalSpaceCache(store string, nodes []string, database, table string, ttl time.Duration, size int) CreatePersonalSpaceCache {
|
||||
func NewCreatePersonalSpaceCache(cfg Config) CreatePersonalSpaceCache {
|
||||
c := &createPersonalSpaceCache{}
|
||||
c.s = getStore(store, nodes, database, table, ttl, size)
|
||||
c.database = database
|
||||
c.table = table
|
||||
c.ttl = ttl
|
||||
c.s = getStore(cfg)
|
||||
c.database = cfg.Database
|
||||
c.table = cfg.Table
|
||||
c.ttl = cfg.TTL
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
14
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/filemetadata.go
generated
vendored
14
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/filemetadata.go
generated
vendored
@@ -18,21 +18,17 @@
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type fileMetadataCache struct {
|
||||
cacheStore
|
||||
}
|
||||
|
||||
// NewFileMetadataCache creates a new FileMetadataCache
|
||||
func NewFileMetadataCache(store string, nodes []string, database, table string, ttl time.Duration, size int) FileMetadataCache {
|
||||
func NewFileMetadataCache(cfg Config) FileMetadataCache {
|
||||
c := &fileMetadataCache{}
|
||||
c.s = getStore(store, nodes, database, table, ttl, size)
|
||||
c.database = database
|
||||
c.table = table
|
||||
c.ttl = ttl
|
||||
c.s = getStore(cfg)
|
||||
c.database = cfg.Database
|
||||
c.table = cfg.Table
|
||||
c.ttl = cfg.TTL
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
11
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/provider.go
generated
vendored
11
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/provider.go
generated
vendored
@@ -20,7 +20,6 @@ package cache
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
@@ -33,12 +32,12 @@ type providerCache struct {
|
||||
}
|
||||
|
||||
// NewProviderCache creates a new ProviderCache
|
||||
func NewProviderCache(store string, nodes []string, database, table string, ttl time.Duration, size int) ProviderCache {
|
||||
func NewProviderCache(cfg Config) ProviderCache {
|
||||
c := &providerCache{}
|
||||
c.s = getStore(store, nodes, database, table, ttl, size)
|
||||
c.database = database
|
||||
c.table = table
|
||||
c.ttl = ttl
|
||||
c.s = getStore(cfg)
|
||||
c.database = cfg.Database
|
||||
c.table = cfg.Table
|
||||
c.ttl = cfg.TTL
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
11
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/stat.go
generated
vendored
11
vendor/github.com/cs3org/reva/v2/pkg/storage/cache/stat.go
generated
vendored
@@ -22,7 +22,6 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
@@ -40,12 +39,12 @@ func init() {
|
||||
}
|
||||
|
||||
// NewStatCache creates a new StatCache
|
||||
func NewStatCache(store string, nodes []string, database, table string, ttl time.Duration, size int) StatCache {
|
||||
func NewStatCache(cfg Config) StatCache {
|
||||
c := statCache{}
|
||||
c.s = getStore(store, nodes, database, table, ttl, size)
|
||||
c.database = database
|
||||
c.table = table
|
||||
c.ttl = ttl
|
||||
c.s = getStore(cfg)
|
||||
c.database = cfg.Database
|
||||
c.table = cfg.Table
|
||||
c.ttl = cfg.TTL
|
||||
return &c
|
||||
}
|
||||
|
||||
|
||||
5
vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go
generated
vendored
5
vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go
generated
vendored
@@ -138,11 +138,12 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) (
|
||||
|
||||
tp := tree.New(lu, bs, o, store.Create(
|
||||
store.Store(o.IDCache.Store),
|
||||
store.TTL(time.Duration(o.IDCache.TTL)*time.Second),
|
||||
store.TTL(o.IDCache.TTL),
|
||||
store.Size(o.IDCache.Size),
|
||||
microstore.Nodes(o.IDCache.Nodes...),
|
||||
microstore.Database(o.IDCache.Database),
|
||||
microstore.Table(o.IDCache.Table),
|
||||
store.DisablePersistence(o.IDCache.DisablePersistence),
|
||||
))
|
||||
|
||||
permissionsSelector, err := pool.PermissionsSelector(o.PermissionsSVC, pool.WithTLSMode(o.PermTLSMode))
|
||||
@@ -203,7 +204,7 @@ func New(o *options.Options, lu *lookup.Lookup, p Permissions, tp Tree, es event
|
||||
p: p,
|
||||
chunkHandler: chunking.NewChunkHandler(filepath.Join(o.Root, "uploads")),
|
||||
stream: es,
|
||||
cache: cache.GetStatCache(o.StatCache.Store, o.StatCache.Nodes, o.StatCache.Database, "stat", time.Duration(o.StatCache.TTL)*time.Second, o.StatCache.Size),
|
||||
cache: cache.GetStatCache(o.StatCache),
|
||||
UserCache: ttlcache.NewCache(),
|
||||
userSpaceIndex: userSpaceIndex,
|
||||
groupSpaceIndex: groupSpaceIndex,
|
||||
|
||||
@@ -27,7 +27,6 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/storage/cache"
|
||||
"github.com/google/renameio/v2"
|
||||
@@ -53,7 +52,7 @@ type readWriteCloseSeekTruncater interface {
|
||||
func NewMessagePackBackend(rootPath string, o cache.Config) MessagePackBackend {
|
||||
return MessagePackBackend{
|
||||
rootPath: filepath.Clean(rootPath),
|
||||
metaCache: cache.GetFileMetadataCache(o.Store, o.Nodes, o.Database, "filemetadata:", time.Duration(o.TTL)*time.Second, o.Size),
|
||||
metaCache: cache.GetFileMetadataCache(o),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
14
vendor/github.com/cs3org/reva/v2/pkg/store/options.go
generated
vendored
14
vendor/github.com/cs3org/reva/v2/pkg/store/options.go
generated
vendored
@@ -75,3 +75,17 @@ func TTL(val time.Duration) store.Option {
|
||||
o.Context = context.WithValue(o.Context, ttlContextKey{}, val)
|
||||
}
|
||||
}
|
||||
|
||||
type disablePersistanceContextKey struct{}
|
||||
|
||||
// DisablePersistence disables the persistence of the store by instructing it to use memory only.
|
||||
// Only supported by the `natsjs` and `natsjskv` implementations.
|
||||
func DisablePersistence(val bool) store.Option {
|
||||
return func(o *store.Options) {
|
||||
if o.Context == nil {
|
||||
o.Context = context.Background()
|
||||
}
|
||||
|
||||
o.Context = context.WithValue(o.Context, disablePersistanceContextKey{}, val)
|
||||
}
|
||||
}
|
||||
|
||||
7
vendor/github.com/cs3org/reva/v2/pkg/store/store.go
generated
vendored
7
vendor/github.com/cs3org/reva/v2/pkg/store/store.go
generated
vendored
@@ -119,6 +119,9 @@ func Create(opts ...microstore.Option) microstore.Store {
|
||||
return *ocMemStore
|
||||
case TypeNatsJS:
|
||||
ttl, _ := options.Context.Value(ttlContextKey{}).(time.Duration)
|
||||
if mem, _ := options.Context.Value(disablePersistanceContextKey{}).(bool); mem {
|
||||
opts = append(opts, natsjs.DefaultMemory())
|
||||
}
|
||||
// TODO nats needs a DefaultTTL option as it does not support per Write TTL ...
|
||||
// FIXME nats has restrictions on the key, we cannot use slashes AFAICT
|
||||
// host, port, clusterid
|
||||
@@ -132,6 +135,10 @@ func Create(opts ...microstore.Option) microstore.Store {
|
||||
case TypeNatsJSKV:
|
||||
// NOTE: nats needs a DefaultTTL option as it does not support per Write TTL ...
|
||||
ttl, _ := options.Context.Value(ttlContextKey{}).(time.Duration)
|
||||
if mem, _ := options.Context.Value(disablePersistanceContextKey{}).(bool); mem {
|
||||
opts = append(opts, natsjskv.DefaultMemory())
|
||||
}
|
||||
|
||||
natsOptions := nats.GetDefaultOptions()
|
||||
natsOptions.Name = "TODO" // we can pass in the service name to allow identifying the client, but that requires adding a custom context option
|
||||
return natsjskv.NewStore(
|
||||
|
||||
2
vendor/modules.txt
vendored
2
vendor/modules.txt
vendored
@@ -362,7 +362,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1
|
||||
github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1
|
||||
github.com/cs3org/go-cs3apis/cs3/tx/v1beta1
|
||||
github.com/cs3org/go-cs3apis/cs3/types/v1beta1
|
||||
# github.com/cs3org/reva/v2 v2.17.1-0.20231215110403-2f85376a4d3f
|
||||
# github.com/cs3org/reva/v2 v2.17.1-0.20231215113433-48c0ea55bf47
|
||||
## explicit; go 1.21
|
||||
github.com/cs3org/reva/v2/cmd/revad/internal/grace
|
||||
github.com/cs3org/reva/v2/cmd/revad/runtime
|
||||
|
||||
Reference in New Issue
Block a user