mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-03-14 02:10:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

allowing disabling persistence of caches

Browse Source 
Signed-off-by: jkoberg <jkoberg@owncloud.com>
This commit is contained in:
jkoberg
2023-12-14 16:42:13 +01:00
parent 0f5a3be661
commit 0d1669b3bd
11 changed files with 198 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
changelog/unreleased/allow-disabling-persistent-caches.md Normal file
View File

@@ -0,0 +1,5 @@
Enhancement: Allow inmemory nats-js-kv stores
Adds envvars to keep nats-js-kv stores in memory and not persist them on disc.
https://github.com/owncloud/ocis/pull/7979

22
services/frontend/pkg/config/config.go
View File

@@ -126,16 +126,18 @@ type DataGateway struct {
}
type OCS struct {
Prefix string `yaml:"prefix" env:"FRONTEND_OCS_PREFIX" desc:"URL path prefix for the OCS service. Note that the string must not start with '/'."`
SharePrefix string `yaml:"share_prefix" env:"FRONTEND_OCS_SHARE_PREFIX" desc:"Path prefix for shares as part of an ocis resource. Note that the path must start with '/'."`
HomeNamespace string `yaml:"home_namespace" env:"FRONTEND_OCS_PERSONAL_NAMESPACE" desc:"Homespace namespace identifier."`
AdditionalInfoAttribute string `yaml:"additional_info_attribute" env:"FRONTEND_OCS_ADDITIONAL_INFO_ATTRIBUTE" desc:"Additional information attribute for the user like {{.Mail}}."`
StatCacheType string `yaml:"stat_cache_type" env:"OCIS_CACHE_STORE;FRONTEND_OCS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;FRONTEND_OCS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
StatCacheTable string `yaml:"stat_cache_table" env:"FRONTEND_OCS_STAT_CACHE_TABLE" desc:"The database table the store should use."`
StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;FRONTEND_OCS_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;FRONTEND_OCS_STAT_CACHE_SIZE" desc:"Max number of entries to hold in the cache."`
Prefix string `yaml:"prefix" env:"FRONTEND_OCS_PREFIX" desc:"URL path prefix for the OCS service. Note that the string must not start with '/'."`
SharePrefix string `yaml:"share_prefix" env:"FRONTEND_OCS_SHARE_PREFIX" desc:"Path prefix for shares as part of an ocis resource. Note that the path must start with '/'."`
HomeNamespace string `yaml:"home_namespace" env:"FRONTEND_OCS_PERSONAL_NAMESPACE" desc:"Homespace namespace identifier."`
AdditionalInfoAttribute string `yaml:"additional_info_attribute" env:"FRONTEND_OCS_ADDITIONAL_INFO_ATTRIBUTE" desc:"Additional information attribute for the user like {{.Mail}}."`
StatCacheType string `yaml:"stat_cache_type" env:"OCIS_CACHE_STORE;FRONTEND_OCS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;FRONTEND_OCS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
StatCacheTable string `yaml:"stat_cache_table" env:"FRONTEND_OCS_STAT_CACHE_TABLE" desc:"The database table the store should use."`
StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;FRONTEND_OCS_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;FRONTEND_OCS_STAT_CACHE_SIZE" desc:"Max number of entries to hold in the cache."`
StatCacheDisablePersistence bool `yaml:"stat_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;FRONTEND_OCS_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disable persistence of the cache. Only applies when using the 'nats-js' store type. Defaults to false."`
CacheWarmupDriver string `yaml:"cache_warmup_driver,omitempty"` // not supported by the oCIS product, therefore not part of docs
CacheWarmupDrivers CacheWarmupDrivers `yaml:"cache_warmup_drivers,omitempty"` // not supported by the oCIS product, therefore not part of docs
EnableDenials bool `yaml:"enable_denials" env:"FRONTEND_OCS_ENABLE_DENIALS" desc:"EXPERIMENTAL: enable the feature to deny access on folders."`

21
services/frontend/pkg/revaconfig/config.go
View File

@@ -156,15 +156,18 @@ func FrontendConfigFromStruct(cfg *config.Config, logger log.Logger) (map[string
"insecure": true,
},
"ocs": map[string]interface{}{
"storage_registry_svc": cfg.Reva.Address,
"share_prefix": cfg.OCS.SharePrefix,
"home_namespace": cfg.OCS.HomeNamespace,
"stat_cache_ttl": cfg.OCS.StatCacheTTL / time.Second,
"stat_cache_size": cfg.OCS.StatCacheSize,
"stat_cache_store": cfg.OCS.StatCacheType,
"stat_cache_nodes": cfg.OCS.StatCacheNodes,
"stat_cache_database": cfg.OCS.StatCacheDatabase,
"stat_cache_table": cfg.OCS.StatCacheTable,
"storage_registry_svc": cfg.Reva.Address,
"share_prefix": cfg.OCS.SharePrefix,
"home_namespace": cfg.OCS.HomeNamespace,
"stat_cache_config": map[string]interface{}{
"cache_store": cfg.OCS.StatCacheType,
"cache_nodes": cfg.OCS.StatCacheNodes,
"cache_database": cfg.OCS.StatCacheDatabase,
"cache_table": cfg.OCS.StatCacheTable,
"cache_ttl": cfg.OCS.StatCacheTTL / time.Second,
"cache_size": cfg.OCS.StatCacheSize,
"cache_disable_persistence": cfg.OCS.StatCacheDisablePersistence,
},
"prefix": cfg.OCS.Prefix,
"additional_info_attribute": cfg.OCS.AdditionalInfoAttribute,
"machine_auth_apikey": cfg.MachineAuthAPIKey,

1
services/gateway/README.md
View File

@@ -23,3 +23,4 @@ Note: The gateway service can only be scaled if not using `memory` store and the
Store specific notes:
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
- When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
- When using `nats-js-kv` store it is possible to use `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache entries on disc.

33
services/gateway/pkg/config/config.go
View File

@@ -85,19 +85,22 @@ type StorageRegistry struct {
// Cache holds cache config
type Cache struct {
StatCacheStore string // NOTE: The stat cache is not working atm. Hence we block configuring it
StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
ProviderCacheSize int `yaml:"provider_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_PROVIDER_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
CreateHomeCacheSize int `yaml:"create_home_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_CREATE_HOME_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
StatCacheStore string // NOTE: The stat cache is not working atm. Hence we block configuring it
StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
StatCacheDisablePersistence bool `yaml:"stat_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the stat cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
ProviderCacheSize int `yaml:"provider_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_PROVIDER_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
ProviderCacheDisablePersistence bool `yaml:"provider_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_PROVIDER_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the provider cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
CreateHomeCacheSize int `yaml:"create_home_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_CREATE_HOME_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
CreateHomeCacheDisablePersistence bool `yaml:"create_home_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_CREATE_HOME_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the create home cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
}

43
services/gateway/pkg/revaconfig/config.go
View File

@@ -4,7 +4,6 @@ import (
"encoding/json"
"os"
"strings"
"time"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
@@ -62,21 +61,33 @@ func GatewayConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]i
"transfer_shared_secret": cfg.TransferSecret,
"transfer_expires": cfg.TransferExpires,
// cache and TTLs
"stat_cache_store": cfg.Cache.StatCacheStore,
"stat_cache_nodes": cfg.Cache.StatCacheNodes,
"stat_cache_database": cfg.Cache.StatCacheDatabase,
"stat_cache_ttl": cfg.Cache.StatCacheTTL / time.Second,
"stat_cache_size": cfg.Cache.StatCacheSize,
"provider_cache_store": cfg.Cache.ProviderCacheStore,
"provider_cache_nodes": cfg.Cache.ProviderCacheNodes,
"provider_cache_database": cfg.Cache.ProviderCacheDatabase,
"provider_cache_ttl": cfg.Cache.ProviderCacheTTL / time.Second,
"provider_cache_size": cfg.Cache.ProviderCacheSize,
"create_home_cache_store": cfg.Cache.CreateHomeCacheStore,
"create_home_cache_nodes": cfg.Cache.CreateHomeCacheNodes,
"create_home_cache_database": cfg.Cache.CreateHomeCacheDatabase,
"create_home_cache_ttl": cfg.Cache.CreateHomeCacheTTL / time.Second,
"create_home_cache_size": cfg.Cache.CreateHomeCacheSize,
"stat_cache_config": map[string]interface{}{
"cache_store": cfg.Cache.StatCacheStore,
"cache_nodes": cfg.Cache.StatCacheNodes,
"cache_database": cfg.Cache.StatCacheDatabase,
"cache_table": "stat",
"cache_ttl": cfg.Cache.StatCacheTTL,
"cache_size": cfg.Cache.StatCacheSize,
"cache_disable_persistenc": cfg.Cache.StatCacheDisablePersistence,
},
"provider_cache_config": map[string]interface{}{
"cache_store": cfg.Cache.ProviderCacheStore,
"cache_nodes": cfg.Cache.ProviderCacheNodes,
"cache_database": cfg.Cache.ProviderCacheDatabase,
"cache_table": "provider",
"cache_ttl": cfg.Cache.ProviderCacheTTL,
"cache_size": cfg.Cache.ProviderCacheSize,
"disable_persistence": cfg.Cache.ProviderCacheDisablePersistence,
},
"create_home_cache_config": map[string]interface{}{
"cache_store": cfg.Cache.CreateHomeCacheStore,
"cache_nodes": cfg.Cache.CreateHomeCacheNodes,
"cache_database": cfg.Cache.CreateHomeCacheDatabase,
"cache_table": "create_home",
"cache_ttl": cfg.Cache.CreateHomeCacheTTL,
"cache_size": cfg.Cache.CreateHomeCacheSize,
"cache_disable_persistence": cfg.Cache.CreateHomeCacheDisablePersistence,
},
},
"authregistry": map[string]interface{}{
"driver": "static",

11
services/storage-system/pkg/config/config.go
View File

@@ -87,9 +87,10 @@ type OCISDriver struct {
// Cache holds cache config
type Cache struct {
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_SYSTEM_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_SYSTEM_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_SYSTEM_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_SYSTEM_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_SYSTEM_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_SYSTEM_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_SYSTEM_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_SYSTEM_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_SYSTEM_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
}

11
services/storage-system/pkg/revaconfig/config.go
View File

@@ -162,11 +162,12 @@ func metadataDrivers(cfg *config.Config) map[string]interface{} {
"cache_database": "system",
},
"filemetadatacache": map[string]interface{}{
"cache_store": cfg.FileMetadataCache.Store,
"cache_nodes": cfg.FileMetadataCache.Nodes,
"cache_database": cfg.FileMetadataCache.Database,
"cache_ttl": cfg.FileMetadataCache.TTL / time.Second,
"cache_size": cfg.FileMetadataCache.Size,
"cache_store": cfg.FileMetadataCache.Store,
"cache_nodes": cfg.FileMetadataCache.Nodes,
"cache_database": cfg.FileMetadataCache.Database,
"cache_ttl": cfg.FileMetadataCache.TTL / time.Second,
"cache_size": cfg.FileMetadataCache.Size,
"cache_disable_persistence": cfg.FileMetadataCache.DisablePersistence,
},
},
}

33
services/storage-users/pkg/config/config.go
View File

@@ -176,29 +176,32 @@ type Events struct {
// StatCache holds cache config
type StatCache struct {
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_STAT_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_STAT_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_USERS_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
}
// FilemetadataCache holds cache config
type FilemetadataCache struct {
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_FILEMETADATA_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_FILEMETADATA_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_FILEMETADATA_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_FILEMETADATA_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_FILEMETADATA_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_FILEMETADATA_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_FILEMETADATA_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."`
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_FILEMETADATA_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_USERS_FILEMETADATA_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
}
// IDCache holds cache config
type IDCache struct {
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_ID_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_ID_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_ID_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens have no expiration. Defaults to 300s which is derived from the underlaying package though not explicitely set as default. See the Environment Variable Types description for more details."`
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_ID_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_ID_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_ID_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."`
Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."`
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_ID_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens have no expiration. Defaults to 300s which is derived from the underlaying package though not explicitly set as default. See the Environment Variable Types description for more details."`
Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_ID_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."`
DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_USERS_ID_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."`
}
// S3Driver is the storage driver configuration when using 's3' storage driver

41
services/storage-users/pkg/revaconfig/config.go
View File

@@ -2,8 +2,6 @@
package revaconfig
import (
"time"
"github.com/owncloud/ocis/v2/services/storage-users/pkg/config"
)
@@ -74,28 +72,31 @@ func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} {
"nats_enable_tls": cfg.Events.EnableTLS,
"data_txs": map[string]interface{}{
"simple": map[string]interface{}{
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL / time.Second,
"cache_size": cfg.StatCache.Size,
"cache_table": "stat",
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL,
"cache_size": cfg.StatCache.Size,
"cache_table": "stat",
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
},
"spaces": map[string]interface{}{
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL / time.Second,
"cache_size": cfg.StatCache.Size,
"cache_table": "stat",
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL,
"cache_size": cfg.StatCache.Size,
"cache_table": "stat",
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
},
"tus": map[string]interface{}{
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL / time.Second,
"cache_size": cfg.StatCache.Size,
"cache_table": "stat",
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL,
"cache_size": cfg.StatCache.Size,
"cache_table": "stat",
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
},
},
},

134
services/storage-users/pkg/revaconfig/drivers.go
View File

@@ -1,8 +1,6 @@
package revaconfig
import (
"time"
"github.com/owncloud/ocis/v2/services/storage-users/pkg/config"
)
@@ -135,25 +133,28 @@ func Ocis(cfg *config.Config) map[string]interface{} {
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
"max_quota": cfg.Drivers.OCIS.MaxQuota,
"statcache": map[string]interface{}{
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL / time.Second,
"cache_size": cfg.StatCache.Size,
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL,
"cache_size": cfg.StatCache.Size,
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
},
"filemetadatacache": map[string]interface{}{
"cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes,
"cache_database": cfg.FilemetadataCache.Database,
"cache_ttl": cfg.FilemetadataCache.TTL / time.Second,
"cache_size": cfg.FilemetadataCache.Size,
"cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes,
"cache_database": cfg.FilemetadataCache.Database,
"cache_ttl": cfg.FilemetadataCache.TTL,
"cache_size": cfg.FilemetadataCache.Size,
"cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence,
},
"idcache": map[string]interface{}{
"cache_store": cfg.IDCache.Store,
"cache_nodes": cfg.IDCache.Nodes,
"cache_database": cfg.IDCache.Database,
"cache_ttl": cfg.IDCache.TTL / time.Second,
"cache_size": cfg.IDCache.Size,
"cache_store": cfg.IDCache.Store,
"cache_nodes": cfg.IDCache.Nodes,
"cache_database": cfg.IDCache.Database,
"cache_ttl": cfg.IDCache.TTL,
"cache_size": cfg.IDCache.Size,
"cache_disable_persistence": cfg.IDCache.DisablePersistence,
},
"events": map[string]interface{}{
"natsaddress": cfg.Events.Addr,
@@ -193,25 +194,28 @@ func OcisNoEvents(cfg *config.Config) map[string]interface{} {
"max_concurrency": cfg.Drivers.OCIS.MaxConcurrency,
"max_quota": cfg.Drivers.OCIS.MaxQuota,
"statcache": map[string]interface{}{
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL / time.Second,
"cache_size": cfg.StatCache.Size,
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL,
"cache_size": cfg.StatCache.Size,
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
},
"filemetadatacache": map[string]interface{}{
"cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes,
"cache_database": cfg.FilemetadataCache.Database,
"cache_ttl": cfg.FilemetadataCache.TTL / time.Second,
"cache_size": cfg.FilemetadataCache.Size,
"cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes,
"cache_database": cfg.FilemetadataCache.Database,
"cache_ttl": cfg.FilemetadataCache.TTL,
"cache_size": cfg.FilemetadataCache.Size,
"cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence,
},
"idcache": map[string]interface{}{
"cache_store": cfg.IDCache.Store,
"cache_nodes": cfg.IDCache.Nodes,
"cache_database": cfg.IDCache.Database,
"cache_ttl": cfg.IDCache.TTL / time.Second,
"cache_size": cfg.IDCache.Size,
"cache_store": cfg.IDCache.Store,
"cache_nodes": cfg.IDCache.Nodes,
"cache_database": cfg.IDCache.Database,
"cache_ttl": cfg.IDCache.TTL,
"cache_size": cfg.IDCache.Size,
"cache_disable_persistence": cfg.IDCache.DisablePersistence,
},
}
}
@@ -256,25 +260,28 @@ func S3NG(cfg *config.Config) map[string]interface{} {
"max_concurrency": cfg.Drivers.S3NG.MaxConcurrency,
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
"statcache": map[string]interface{}{
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL / time.Second,
"cache_size": cfg.StatCache.Size,
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL,
"cache_size": cfg.StatCache.Size,
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
},
"filemetadatacache": map[string]interface{}{
"cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes,
"cache_database": cfg.FilemetadataCache.Database,
"cache_ttl": cfg.FilemetadataCache.TTL / time.Second,
"cache_size": cfg.FilemetadataCache.Size,
"cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes,
"cache_database": cfg.FilemetadataCache.Database,
"cache_ttl": cfg.FilemetadataCache.TTL,
"cache_size": cfg.FilemetadataCache.Size,
"cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence,
},
"idcache": map[string]interface{}{
"cache_store": cfg.IDCache.Store,
"cache_nodes": cfg.IDCache.Nodes,
"cache_database": cfg.IDCache.Database,
"cache_ttl": cfg.IDCache.TTL / time.Second,
"cache_size": cfg.IDCache.Size,
"cache_store": cfg.IDCache.Store,
"cache_nodes": cfg.IDCache.Nodes,
"cache_database": cfg.IDCache.Database,
"cache_ttl": cfg.IDCache.TTL,
"cache_size": cfg.IDCache.Size,
"cache_disable_persistence": cfg.IDCache.DisablePersistence,
},
"events": map[string]interface{}{
"natsaddress": cfg.Events.Addr,
@@ -318,25 +325,28 @@ func S3NGNoEvents(cfg *config.Config) map[string]interface{} {
"max_concurrency": cfg.Drivers.S3NG.MaxConcurrency,
"lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor,
"statcache": map[string]interface{}{
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL / time.Second,
"cache_size": cfg.StatCache.Size,
"cache_store": cfg.StatCache.Store,
"cache_nodes": cfg.StatCache.Nodes,
"cache_database": cfg.StatCache.Database,
"cache_ttl": cfg.StatCache.TTL,
"cache_size": cfg.StatCache.Size,
"cache_disable_persistence": cfg.StatCache.DisablePersistence,
},
"filemetadatacache": map[string]interface{}{
"cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes,
"cache_database": cfg.FilemetadataCache.Database,
"cache_ttl": cfg.FilemetadataCache.TTL / time.Second,
"cache_size": cfg.FilemetadataCache.Size,
"cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes,
"cache_database": cfg.FilemetadataCache.Database,
"cache_ttl": cfg.FilemetadataCache.TTL,
"cache_size": cfg.FilemetadataCache.Size,
"cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence,
},
"idcache": map[string]interface{}{
"cache_store": cfg.IDCache.Store,
"cache_nodes": cfg.IDCache.Nodes,
"cache_database": cfg.IDCache.Database,
"cache_ttl": cfg.IDCache.TTL / time.Second,
"cache_size": cfg.IDCache.Size,
"cache_store": cfg.IDCache.Store,
"cache_nodes": cfg.IDCache.Nodes,
"cache_database": cfg.IDCache.Database,
"cache_ttl": cfg.IDCache.TTL,
"cache_size": cfg.IDCache.Size,
"cache_disable_persistence": cfg.IDCache.DisablePersistence,
},
}
}