diff --git a/changelog/unreleased/allow-disabling-persistent-caches.md b/changelog/unreleased/allow-disabling-persistent-caches.md new file mode 100644 index 0000000000..d6b1e43f17 --- /dev/null +++ b/changelog/unreleased/allow-disabling-persistent-caches.md @@ -0,0 +1,5 @@ +Enhancement: Allow inmemory nats-js-kv stores + +Adds envvars to keep nats-js-kv stores in memory and not persist them on disc. + +https://github.com/owncloud/ocis/pull/7979 diff --git a/services/frontend/pkg/config/config.go b/services/frontend/pkg/config/config.go index 42262ebf51..f8c12cd80c 100644 --- a/services/frontend/pkg/config/config.go +++ b/services/frontend/pkg/config/config.go @@ -126,16 +126,18 @@ type DataGateway struct { } type OCS struct { - Prefix string `yaml:"prefix" env:"FRONTEND_OCS_PREFIX" desc:"URL path prefix for the OCS service. Note that the string must not start with '/'."` - SharePrefix string `yaml:"share_prefix" env:"FRONTEND_OCS_SHARE_PREFIX" desc:"Path prefix for shares as part of an ocis resource. Note that the path must start with '/'."` - HomeNamespace string `yaml:"home_namespace" env:"FRONTEND_OCS_PERSONAL_NAMESPACE" desc:"Homespace namespace identifier."` - AdditionalInfoAttribute string `yaml:"additional_info_attribute" env:"FRONTEND_OCS_ADDITIONAL_INFO_ATTRIBUTE" desc:"Additional information attribute for the user like {{.Mail}}."` - StatCacheType string `yaml:"stat_cache_type" env:"OCIS_CACHE_STORE;FRONTEND_OCS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` - StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;FRONTEND_OCS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` - StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - StatCacheTable string `yaml:"stat_cache_table" env:"FRONTEND_OCS_STAT_CACHE_TABLE" desc:"The database table the store should use."` - StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;FRONTEND_OCS_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` - StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;FRONTEND_OCS_STAT_CACHE_SIZE" desc:"Max number of entries to hold in the cache."` + Prefix string `yaml:"prefix" env:"FRONTEND_OCS_PREFIX" desc:"URL path prefix for the OCS service. Note that the string must not start with '/'."` + SharePrefix string `yaml:"share_prefix" env:"FRONTEND_OCS_SHARE_PREFIX" desc:"Path prefix for shares as part of an ocis resource. Note that the path must start with '/'."` + HomeNamespace string `yaml:"home_namespace" env:"FRONTEND_OCS_PERSONAL_NAMESPACE" desc:"Homespace namespace identifier."` + AdditionalInfoAttribute string `yaml:"additional_info_attribute" env:"FRONTEND_OCS_ADDITIONAL_INFO_ATTRIBUTE" desc:"Additional information attribute for the user like {{.Mail}}."` + StatCacheType string `yaml:"stat_cache_type" env:"OCIS_CACHE_STORE;FRONTEND_OCS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;FRONTEND_OCS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` + StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + StatCacheTable string `yaml:"stat_cache_table" env:"FRONTEND_OCS_STAT_CACHE_TABLE" desc:"The database table the store should use."` + StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;FRONTEND_OCS_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` + StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;FRONTEND_OCS_STAT_CACHE_SIZE" desc:"Max number of entries to hold in the cache."` + StatCacheDisablePersistence bool `yaml:"stat_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;FRONTEND_OCS_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disable persistence of the cache. Only applies when using the 'nats-js' store type. Defaults to false."` + CacheWarmupDriver string `yaml:"cache_warmup_driver,omitempty"` // not supported by the oCIS product, therefore not part of docs CacheWarmupDrivers CacheWarmupDrivers `yaml:"cache_warmup_drivers,omitempty"` // not supported by the oCIS product, therefore not part of docs EnableDenials bool `yaml:"enable_denials" env:"FRONTEND_OCS_ENABLE_DENIALS" desc:"EXPERIMENTAL: enable the feature to deny access on folders."` diff --git a/services/frontend/pkg/revaconfig/config.go b/services/frontend/pkg/revaconfig/config.go index 38dadd16b2..3f9489f2dc 100644 --- a/services/frontend/pkg/revaconfig/config.go +++ b/services/frontend/pkg/revaconfig/config.go @@ -156,15 +156,18 @@ func FrontendConfigFromStruct(cfg *config.Config, logger log.Logger) (map[string "insecure": true, }, "ocs": map[string]interface{}{ - "storage_registry_svc": cfg.Reva.Address, - "share_prefix": cfg.OCS.SharePrefix, - "home_namespace": cfg.OCS.HomeNamespace, - "stat_cache_ttl": cfg.OCS.StatCacheTTL / time.Second, - "stat_cache_size": cfg.OCS.StatCacheSize, - "stat_cache_store": cfg.OCS.StatCacheType, - "stat_cache_nodes": cfg.OCS.StatCacheNodes, - "stat_cache_database": cfg.OCS.StatCacheDatabase, - "stat_cache_table": cfg.OCS.StatCacheTable, + "storage_registry_svc": cfg.Reva.Address, + "share_prefix": cfg.OCS.SharePrefix, + "home_namespace": cfg.OCS.HomeNamespace, + "stat_cache_config": map[string]interface{}{ + "cache_store": cfg.OCS.StatCacheType, + "cache_nodes": cfg.OCS.StatCacheNodes, + "cache_database": cfg.OCS.StatCacheDatabase, + "cache_table": cfg.OCS.StatCacheTable, + "cache_ttl": cfg.OCS.StatCacheTTL / time.Second, + "cache_size": cfg.OCS.StatCacheSize, + "cache_disable_persistence": cfg.OCS.StatCacheDisablePersistence, + }, "prefix": cfg.OCS.Prefix, "additional_info_attribute": cfg.OCS.AdditionalInfoAttribute, "machine_auth_apikey": cfg.MachineAuthAPIKey, diff --git a/services/gateway/README.md b/services/gateway/README.md index 4ded248260..af5c211886 100644 --- a/services/gateway/README.md +++ b/services/gateway/README.md @@ -23,3 +23,4 @@ Note: The gateway service can only be scaled if not using `memory` store and the Store specific notes: - When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `:/` like `10.10.0.200:26379/mymaster`. - When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus. + - When using `nats-js-kv` store it is possible to use `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache entries on disc. diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index b9393ca028..0de7d6bb78 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -85,19 +85,22 @@ type StorageRegistry struct { // Cache holds cache config type Cache struct { - StatCacheStore string // NOTE: The stat cache is not working atm. Hence we block configuring it - StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` - StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` - StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` - ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` - ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` - ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` - ProviderCacheSize int `yaml:"provider_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_PROVIDER_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` - CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` - CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` - CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` - CreateHomeCacheSize int `yaml:"create_home_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_CREATE_HOME_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + StatCacheStore string // NOTE: The stat cache is not working atm. Hence we block configuring it + StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` + StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` + StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + StatCacheDisablePersistence bool `yaml:"stat_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the stat cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."` + ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` + ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` + ProviderCacheSize int `yaml:"provider_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_PROVIDER_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + ProviderCacheDisablePersistence bool `yaml:"provider_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_PROVIDER_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the provider cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."` + CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` + CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` + CreateHomeCacheSize int `yaml:"create_home_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_CREATE_HOME_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + CreateHomeCacheDisablePersistence bool `yaml:"create_home_cache_disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;GATEWAY_CREATE_HOME_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the create home cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."` } diff --git a/services/gateway/pkg/revaconfig/config.go b/services/gateway/pkg/revaconfig/config.go index 252a1364ee..d1affa7a92 100644 --- a/services/gateway/pkg/revaconfig/config.go +++ b/services/gateway/pkg/revaconfig/config.go @@ -4,7 +4,6 @@ import ( "encoding/json" "os" "strings" - "time" "github.com/cs3org/reva/v2/pkg/utils" "github.com/owncloud/ocis/v2/ocis-pkg/log" @@ -62,21 +61,33 @@ func GatewayConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]i "transfer_shared_secret": cfg.TransferSecret, "transfer_expires": cfg.TransferExpires, // cache and TTLs - "stat_cache_store": cfg.Cache.StatCacheStore, - "stat_cache_nodes": cfg.Cache.StatCacheNodes, - "stat_cache_database": cfg.Cache.StatCacheDatabase, - "stat_cache_ttl": cfg.Cache.StatCacheTTL / time.Second, - "stat_cache_size": cfg.Cache.StatCacheSize, - "provider_cache_store": cfg.Cache.ProviderCacheStore, - "provider_cache_nodes": cfg.Cache.ProviderCacheNodes, - "provider_cache_database": cfg.Cache.ProviderCacheDatabase, - "provider_cache_ttl": cfg.Cache.ProviderCacheTTL / time.Second, - "provider_cache_size": cfg.Cache.ProviderCacheSize, - "create_home_cache_store": cfg.Cache.CreateHomeCacheStore, - "create_home_cache_nodes": cfg.Cache.CreateHomeCacheNodes, - "create_home_cache_database": cfg.Cache.CreateHomeCacheDatabase, - "create_home_cache_ttl": cfg.Cache.CreateHomeCacheTTL / time.Second, - "create_home_cache_size": cfg.Cache.CreateHomeCacheSize, + "stat_cache_config": map[string]interface{}{ + "cache_store": cfg.Cache.StatCacheStore, + "cache_nodes": cfg.Cache.StatCacheNodes, + "cache_database": cfg.Cache.StatCacheDatabase, + "cache_table": "stat", + "cache_ttl": cfg.Cache.StatCacheTTL, + "cache_size": cfg.Cache.StatCacheSize, + "cache_disable_persistenc": cfg.Cache.StatCacheDisablePersistence, + }, + "provider_cache_config": map[string]interface{}{ + "cache_store": cfg.Cache.ProviderCacheStore, + "cache_nodes": cfg.Cache.ProviderCacheNodes, + "cache_database": cfg.Cache.ProviderCacheDatabase, + "cache_table": "provider", + "cache_ttl": cfg.Cache.ProviderCacheTTL, + "cache_size": cfg.Cache.ProviderCacheSize, + "disable_persistence": cfg.Cache.ProviderCacheDisablePersistence, + }, + "create_home_cache_config": map[string]interface{}{ + "cache_store": cfg.Cache.CreateHomeCacheStore, + "cache_nodes": cfg.Cache.CreateHomeCacheNodes, + "cache_database": cfg.Cache.CreateHomeCacheDatabase, + "cache_table": "create_home", + "cache_ttl": cfg.Cache.CreateHomeCacheTTL, + "cache_size": cfg.Cache.CreateHomeCacheSize, + "cache_disable_persistence": cfg.Cache.CreateHomeCacheDisablePersistence, + }, }, "authregistry": map[string]interface{}{ "driver": "static", diff --git a/services/storage-system/pkg/config/config.go b/services/storage-system/pkg/config/config.go index 83cbf29a50..a72d84ced7 100644 --- a/services/storage-system/pkg/config/config.go +++ b/services/storage-system/pkg/config/config.go @@ -87,9 +87,10 @@ type OCISDriver struct { // Cache holds cache config type Cache struct { - Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_SYSTEM_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` - Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_SYSTEM_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` - Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_SYSTEM_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` - Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_SYSTEM_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_SYSTEM_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_SYSTEM_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` + Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_SYSTEM_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` + Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_SYSTEM_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_SYSTEM_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."` } diff --git a/services/storage-system/pkg/revaconfig/config.go b/services/storage-system/pkg/revaconfig/config.go index 5a2e1a51d2..16c1c3a09d 100644 --- a/services/storage-system/pkg/revaconfig/config.go +++ b/services/storage-system/pkg/revaconfig/config.go @@ -162,11 +162,12 @@ func metadataDrivers(cfg *config.Config) map[string]interface{} { "cache_database": "system", }, "filemetadatacache": map[string]interface{}{ - "cache_store": cfg.FileMetadataCache.Store, - "cache_nodes": cfg.FileMetadataCache.Nodes, - "cache_database": cfg.FileMetadataCache.Database, - "cache_ttl": cfg.FileMetadataCache.TTL / time.Second, - "cache_size": cfg.FileMetadataCache.Size, + "cache_store": cfg.FileMetadataCache.Store, + "cache_nodes": cfg.FileMetadataCache.Nodes, + "cache_database": cfg.FileMetadataCache.Database, + "cache_ttl": cfg.FileMetadataCache.TTL / time.Second, + "cache_size": cfg.FileMetadataCache.Size, + "cache_disable_persistence": cfg.FileMetadataCache.DisablePersistence, }, }, } diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 4070394e98..336307aecf 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -176,29 +176,32 @@ type Events struct { // StatCache holds cache config type StatCache struct { - Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` - Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` - Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_STAT_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` - Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` + Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_STAT_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` + Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_USERS_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."` } // FilemetadataCache holds cache config type FilemetadataCache struct { - Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_FILEMETADATA_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` - Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_FILEMETADATA_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` - Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_FILEMETADATA_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` - Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_FILEMETADATA_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_FILEMETADATA_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_FILEMETADATA_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` + Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_FILEMETADATA_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details."` + Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_FILEMETADATA_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_USERS_FILEMETADATA_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."` } // IDCache holds cache config type IDCache struct { - Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_ID_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` - Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_ID_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` - Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_ID_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens have no expiration. Defaults to 300s which is derived from the underlaying package though not explicitely set as default. See the Environment Variable Types description for more details."` - Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_ID_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_ID_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_ID_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` + Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_ID_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens have no expiration. Defaults to 300s which is derived from the underlaying package though not explicitly set as default. See the Environment Variable Types description for more details."` + Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_ID_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default."` + DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;STORAGE_USERS_ID_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false."` } // S3Driver is the storage driver configuration when using 's3' storage driver diff --git a/services/storage-users/pkg/revaconfig/config.go b/services/storage-users/pkg/revaconfig/config.go index ea70cd8500..3966d86c56 100644 --- a/services/storage-users/pkg/revaconfig/config.go +++ b/services/storage-users/pkg/revaconfig/config.go @@ -2,8 +2,6 @@ package revaconfig import ( - "time" - "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" ) @@ -74,28 +72,31 @@ func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} { "nats_enable_tls": cfg.Events.EnableTLS, "data_txs": map[string]interface{}{ "simple": map[string]interface{}{ - "cache_store": cfg.StatCache.Store, - "cache_nodes": cfg.StatCache.Nodes, - "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL / time.Second, - "cache_size": cfg.StatCache.Size, - "cache_table": "stat", + "cache_store": cfg.StatCache.Store, + "cache_nodes": cfg.StatCache.Nodes, + "cache_database": cfg.StatCache.Database, + "cache_ttl": cfg.StatCache.TTL, + "cache_size": cfg.StatCache.Size, + "cache_table": "stat", + "cache_disable_persistence": cfg.StatCache.DisablePersistence, }, "spaces": map[string]interface{}{ - "cache_store": cfg.StatCache.Store, - "cache_nodes": cfg.StatCache.Nodes, - "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL / time.Second, - "cache_size": cfg.StatCache.Size, - "cache_table": "stat", + "cache_store": cfg.StatCache.Store, + "cache_nodes": cfg.StatCache.Nodes, + "cache_database": cfg.StatCache.Database, + "cache_ttl": cfg.StatCache.TTL, + "cache_size": cfg.StatCache.Size, + "cache_table": "stat", + "cache_disable_persistence": cfg.StatCache.DisablePersistence, }, "tus": map[string]interface{}{ - "cache_store": cfg.StatCache.Store, - "cache_nodes": cfg.StatCache.Nodes, - "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL / time.Second, - "cache_size": cfg.StatCache.Size, - "cache_table": "stat", + "cache_store": cfg.StatCache.Store, + "cache_nodes": cfg.StatCache.Nodes, + "cache_database": cfg.StatCache.Database, + "cache_ttl": cfg.StatCache.TTL, + "cache_size": cfg.StatCache.Size, + "cache_table": "stat", + "cache_disable_persistence": cfg.StatCache.DisablePersistence, }, }, }, diff --git a/services/storage-users/pkg/revaconfig/drivers.go b/services/storage-users/pkg/revaconfig/drivers.go index 097f16250f..a26bf8de47 100644 --- a/services/storage-users/pkg/revaconfig/drivers.go +++ b/services/storage-users/pkg/revaconfig/drivers.go @@ -1,8 +1,6 @@ package revaconfig import ( - "time" - "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" ) @@ -135,25 +133,28 @@ func Ocis(cfg *config.Config) map[string]interface{} { "asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads, "max_quota": cfg.Drivers.OCIS.MaxQuota, "statcache": map[string]interface{}{ - "cache_store": cfg.StatCache.Store, - "cache_nodes": cfg.StatCache.Nodes, - "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL / time.Second, - "cache_size": cfg.StatCache.Size, + "cache_store": cfg.StatCache.Store, + "cache_nodes": cfg.StatCache.Nodes, + "cache_database": cfg.StatCache.Database, + "cache_ttl": cfg.StatCache.TTL, + "cache_size": cfg.StatCache.Size, + "cache_disable_persistence": cfg.StatCache.DisablePersistence, }, "filemetadatacache": map[string]interface{}{ - "cache_store": cfg.FilemetadataCache.Store, - "cache_nodes": cfg.FilemetadataCache.Nodes, - "cache_database": cfg.FilemetadataCache.Database, - "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, - "cache_size": cfg.FilemetadataCache.Size, + "cache_store": cfg.FilemetadataCache.Store, + "cache_nodes": cfg.FilemetadataCache.Nodes, + "cache_database": cfg.FilemetadataCache.Database, + "cache_ttl": cfg.FilemetadataCache.TTL, + "cache_size": cfg.FilemetadataCache.Size, + "cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence, }, "idcache": map[string]interface{}{ - "cache_store": cfg.IDCache.Store, - "cache_nodes": cfg.IDCache.Nodes, - "cache_database": cfg.IDCache.Database, - "cache_ttl": cfg.IDCache.TTL / time.Second, - "cache_size": cfg.IDCache.Size, + "cache_store": cfg.IDCache.Store, + "cache_nodes": cfg.IDCache.Nodes, + "cache_database": cfg.IDCache.Database, + "cache_ttl": cfg.IDCache.TTL, + "cache_size": cfg.IDCache.Size, + "cache_disable_persistence": cfg.IDCache.DisablePersistence, }, "events": map[string]interface{}{ "natsaddress": cfg.Events.Addr, @@ -193,25 +194,28 @@ func OcisNoEvents(cfg *config.Config) map[string]interface{} { "max_concurrency": cfg.Drivers.OCIS.MaxConcurrency, "max_quota": cfg.Drivers.OCIS.MaxQuota, "statcache": map[string]interface{}{ - "cache_store": cfg.StatCache.Store, - "cache_nodes": cfg.StatCache.Nodes, - "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL / time.Second, - "cache_size": cfg.StatCache.Size, + "cache_store": cfg.StatCache.Store, + "cache_nodes": cfg.StatCache.Nodes, + "cache_database": cfg.StatCache.Database, + "cache_ttl": cfg.StatCache.TTL, + "cache_size": cfg.StatCache.Size, + "cache_disable_persistence": cfg.StatCache.DisablePersistence, }, "filemetadatacache": map[string]interface{}{ - "cache_store": cfg.FilemetadataCache.Store, - "cache_nodes": cfg.FilemetadataCache.Nodes, - "cache_database": cfg.FilemetadataCache.Database, - "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, - "cache_size": cfg.FilemetadataCache.Size, + "cache_store": cfg.FilemetadataCache.Store, + "cache_nodes": cfg.FilemetadataCache.Nodes, + "cache_database": cfg.FilemetadataCache.Database, + "cache_ttl": cfg.FilemetadataCache.TTL, + "cache_size": cfg.FilemetadataCache.Size, + "cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence, }, "idcache": map[string]interface{}{ - "cache_store": cfg.IDCache.Store, - "cache_nodes": cfg.IDCache.Nodes, - "cache_database": cfg.IDCache.Database, - "cache_ttl": cfg.IDCache.TTL / time.Second, - "cache_size": cfg.IDCache.Size, + "cache_store": cfg.IDCache.Store, + "cache_nodes": cfg.IDCache.Nodes, + "cache_database": cfg.IDCache.Database, + "cache_ttl": cfg.IDCache.TTL, + "cache_size": cfg.IDCache.Size, + "cache_disable_persistence": cfg.IDCache.DisablePersistence, }, } } @@ -256,25 +260,28 @@ func S3NG(cfg *config.Config) map[string]interface{} { "max_concurrency": cfg.Drivers.S3NG.MaxConcurrency, "asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads, "statcache": map[string]interface{}{ - "cache_store": cfg.StatCache.Store, - "cache_nodes": cfg.StatCache.Nodes, - "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL / time.Second, - "cache_size": cfg.StatCache.Size, + "cache_store": cfg.StatCache.Store, + "cache_nodes": cfg.StatCache.Nodes, + "cache_database": cfg.StatCache.Database, + "cache_ttl": cfg.StatCache.TTL, + "cache_size": cfg.StatCache.Size, + "cache_disable_persistence": cfg.StatCache.DisablePersistence, }, "filemetadatacache": map[string]interface{}{ - "cache_store": cfg.FilemetadataCache.Store, - "cache_nodes": cfg.FilemetadataCache.Nodes, - "cache_database": cfg.FilemetadataCache.Database, - "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, - "cache_size": cfg.FilemetadataCache.Size, + "cache_store": cfg.FilemetadataCache.Store, + "cache_nodes": cfg.FilemetadataCache.Nodes, + "cache_database": cfg.FilemetadataCache.Database, + "cache_ttl": cfg.FilemetadataCache.TTL, + "cache_size": cfg.FilemetadataCache.Size, + "cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence, }, "idcache": map[string]interface{}{ - "cache_store": cfg.IDCache.Store, - "cache_nodes": cfg.IDCache.Nodes, - "cache_database": cfg.IDCache.Database, - "cache_ttl": cfg.IDCache.TTL / time.Second, - "cache_size": cfg.IDCache.Size, + "cache_store": cfg.IDCache.Store, + "cache_nodes": cfg.IDCache.Nodes, + "cache_database": cfg.IDCache.Database, + "cache_ttl": cfg.IDCache.TTL, + "cache_size": cfg.IDCache.Size, + "cache_disable_persistence": cfg.IDCache.DisablePersistence, }, "events": map[string]interface{}{ "natsaddress": cfg.Events.Addr, @@ -318,25 +325,28 @@ func S3NGNoEvents(cfg *config.Config) map[string]interface{} { "max_concurrency": cfg.Drivers.S3NG.MaxConcurrency, "lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor, "statcache": map[string]interface{}{ - "cache_store": cfg.StatCache.Store, - "cache_nodes": cfg.StatCache.Nodes, - "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL / time.Second, - "cache_size": cfg.StatCache.Size, + "cache_store": cfg.StatCache.Store, + "cache_nodes": cfg.StatCache.Nodes, + "cache_database": cfg.StatCache.Database, + "cache_ttl": cfg.StatCache.TTL, + "cache_size": cfg.StatCache.Size, + "cache_disable_persistence": cfg.StatCache.DisablePersistence, }, "filemetadatacache": map[string]interface{}{ - "cache_store": cfg.FilemetadataCache.Store, - "cache_nodes": cfg.FilemetadataCache.Nodes, - "cache_database": cfg.FilemetadataCache.Database, - "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, - "cache_size": cfg.FilemetadataCache.Size, + "cache_store": cfg.FilemetadataCache.Store, + "cache_nodes": cfg.FilemetadataCache.Nodes, + "cache_database": cfg.FilemetadataCache.Database, + "cache_ttl": cfg.FilemetadataCache.TTL, + "cache_size": cfg.FilemetadataCache.Size, + "cache_disable_persistence": cfg.FilemetadataCache.DisablePersistence, }, "idcache": map[string]interface{}{ - "cache_store": cfg.IDCache.Store, - "cache_nodes": cfg.IDCache.Nodes, - "cache_database": cfg.IDCache.Database, - "cache_ttl": cfg.IDCache.TTL / time.Second, - "cache_size": cfg.IDCache.Size, + "cache_store": cfg.IDCache.Store, + "cache_nodes": cfg.IDCache.Nodes, + "cache_database": cfg.IDCache.Database, + "cache_ttl": cfg.IDCache.TTL, + "cache_size": cfg.IDCache.Size, + "cache_disable_persistence": cfg.IDCache.DisablePersistence, }, } }