mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2025-12-31 01:10:20 -06:00
Proxy accesstoken cache store (#5829)
* refactor middleware options Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * use ocmemstore micro store implementaiton for token cache Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * refactor ocis store options, support redis sentinel Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * align cache configuration Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * database and tabe are used to build prefixes for inmemory stores Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * add global persistent store options to userlog config Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * log cache errors but continue Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * drup unnecessary type conversion Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * Better description for the default userinfo ttl Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * use global cache options for even more caches Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * don't log userinfo cache misses Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * default to stock memory store Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * use correct mem store typo string Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * split cache options, doc cleanup Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * mint and write userinfo to cache async Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * use hashed token as key Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * go mod tidy Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * update docs Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * update cache store naming Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * bring back depreceted ocis-pkg/store package for backwards compatability Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * update changelog Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * Apply suggestions from code review Co-authored-by: kobergj <jkoberg@owncloud.com> * revert ocis-pkg/cache to store rename Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> * add waiting for each step 50 milliseconds * starlack check --------- Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de> Co-authored-by: kobergj <jkoberg@owncloud.com> Co-authored-by: Viktor Scharf <scharf.vi@gmail.com>
This commit is contained in:
Jörn Friedrich Dreyer
committed by
GitHub
GitHub
parent
688d07e297
commit
6bec87f582
@@ -1148,6 +1148,7 @@ def e2eTests(ctx):
|
||||
"RETRY": "1",
|
||||
"WEB_UI_CONFIG": "%s/%s" % (dirs["base"], dirs["ocisConfig"]),
|
||||
"LOCAL_UPLOAD_DIR": "/uploads",
|
||||
"SLOW_MO": "50",
|
||||
},
|
||||
"commands": [
|
||||
"cd %s" % dirs["web"],
|
||||
|
||||
10
changelog/unreleased/change-cache-configuration.md
Normal file
10
changelog/unreleased/change-cache-configuration.md
Normal file
@@ -0,0 +1,10 @@
|
||||
Change: Updated Cache Configuration
|
||||
|
||||
We updated all cache related environment vars to more closely follow the go micro naming pattern:
|
||||
- `{service}_CACHE_STORE_TYPE` becomes `{service}_CACHE_STORE` or `{service}_PERSISTENT_STORE`
|
||||
- `{service}_CACHE_STORE_ADDRESS(ES)` becomes `{service}_CACHE_STORE_NODES`
|
||||
- The `mem` store implementation name changes to `memory`
|
||||
- In yaml files the cache `type` becomes `store`
|
||||
We introduced `redis-sentinel` as a store implementation.
|
||||
|
||||
https://github.com/owncloud/ocis/pull/5829
|
||||
4
go.mod
4
go.mod
@@ -38,6 +38,7 @@ require (
|
||||
github.com/go-micro/plugins/v4/wrapper/monitoring/prometheus v1.2.0
|
||||
github.com/go-micro/plugins/v4/wrapper/trace/opencensus v1.1.0
|
||||
github.com/go-ozzo/ozzo-validation/v4 v4.3.0
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/gofrs/uuid v4.4.0+incompatible
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0
|
||||
github.com/golang/protobuf v1.5.3
|
||||
@@ -69,6 +70,7 @@ require (
|
||||
github.com/prometheus/client_golang v1.14.0
|
||||
github.com/rogpeppe/go-internal v1.8.0
|
||||
github.com/rs/zerolog v1.29.0
|
||||
github.com/shamaton/msgpack/v2 v2.1.1
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/spf13/cobra v1.6.1
|
||||
github.com/stretchr/testify v1.8.2
|
||||
@@ -180,7 +182,6 @@ require (
|
||||
github.com/go-logfmt/logfmt v0.5.1 // indirect
|
||||
github.com/go-logr/logr v1.2.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5 // indirect
|
||||
github.com/go-sql-driver/mysql v1.6.0 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
|
||||
github.com/gobwas/glob v0.2.3 // indirect
|
||||
@@ -274,7 +275,6 @@ require (
|
||||
github.com/sciencemesh/meshdirectory-web v1.0.4 // indirect
|
||||
github.com/sergi/go-diff v1.2.0 // indirect
|
||||
github.com/sethvargo/go-password v0.2.0 // indirect
|
||||
github.com/shamaton/msgpack/v2 v2.1.1 // indirect
|
||||
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
|
||||
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect
|
||||
github.com/sony/gobreaker v0.5.0 // indirect
|
||||
|
||||
@@ -54,7 +54,7 @@ type Config struct {
|
||||
|
||||
Tracing *shared.Tracing `yaml:"tracing"`
|
||||
Log *shared.Log `yaml:"log"`
|
||||
CacheStore *shared.CacheStore `yaml:"cache_store"`
|
||||
Cache *shared.Cache `yaml:"cache"`
|
||||
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
|
||||
GRPCServiceTLS *shared.GRPCServiceTLS `yaml:"grpc_service_tls"`
|
||||
HTTPServiceTLS shared.HTTPServiceTLS `yaml:"http_service_tls"`
|
||||
|
||||
@@ -49,8 +49,8 @@ func EnsureDefaults(cfg *config.Config) {
|
||||
if cfg.TokenManager == nil {
|
||||
cfg.TokenManager = &shared.TokenManager{}
|
||||
}
|
||||
if cfg.CacheStore == nil {
|
||||
cfg.CacheStore = &shared.CacheStore{}
|
||||
if cfg.Cache == nil {
|
||||
cfg.Cache = &shared.Cache{}
|
||||
}
|
||||
if cfg.GRPCClientTLS == nil {
|
||||
cfg.GRPCClientTLS = &shared.GRPCClientTLS{}
|
||||
@@ -70,7 +70,7 @@ func EnsureCommons(cfg *config.Config) {
|
||||
|
||||
cfg.Commons.Log = structs.CopyOrZeroValue(cfg.Log)
|
||||
cfg.Commons.Tracing = structs.CopyOrZeroValue(cfg.Tracing)
|
||||
cfg.Commons.CacheStore = structs.CopyOrZeroValue(cfg.CacheStore)
|
||||
cfg.Commons.Cache = structs.CopyOrZeroValue(cfg.Cache)
|
||||
|
||||
if cfg.GRPCClientTLS != nil {
|
||||
cfg.Commons.GRPCClientTLS = cfg.GRPCClientTLS
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
package roles
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/sync"
|
||||
settingsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0"
|
||||
)
|
||||
|
||||
// cache is a cache implementation for roles, keyed by roleIDs.
|
||||
type cache struct {
|
||||
sc sync.Cache
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
// newCache returns a new instance of Cache.
|
||||
func newCache(capacity int, ttl time.Duration) cache {
|
||||
return cache{
|
||||
ttl: ttl,
|
||||
sc: sync.NewCache(capacity),
|
||||
}
|
||||
}
|
||||
|
||||
// get gets a role-bundle by a given `roleID`.
|
||||
func (c *cache) get(roleID string) *settingsmsg.Bundle {
|
||||
if ce := c.sc.Load(roleID); ce != nil {
|
||||
return ce.V.(*settingsmsg.Bundle)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// set sets a roleID / role-bundle.
|
||||
func (c *cache) set(roleID string, value *settingsmsg.Bundle) {
|
||||
c.sc.Store(roleID, value, time.Now().Add(c.ttl))
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package roles
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
settingsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func cacheRunner(size int, ttl time.Duration) (*cache, func(f func(v string))) {
|
||||
c := newCache(size, ttl)
|
||||
run := func(f func(v string)) {
|
||||
wg := sync.WaitGroup{}
|
||||
for i := 0; i < size; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
f(strconv.Itoa(i))
|
||||
wg.Done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
return &c, run
|
||||
}
|
||||
|
||||
func BenchmarkCache(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
size := 1024
|
||||
c, cr := cacheRunner(size, 100*time.Millisecond)
|
||||
|
||||
cr(func(v string) { c.set(v, &settingsmsg.Bundle{}) })
|
||||
cr(func(v string) { c.get(v) })
|
||||
}
|
||||
|
||||
func TestCache(t *testing.T) {
|
||||
size := 1024
|
||||
ttl := 100 * time.Millisecond
|
||||
c, cr := cacheRunner(size, ttl)
|
||||
|
||||
cr(func(v string) {
|
||||
c.set(v, &settingsmsg.Bundle{Id: v})
|
||||
})
|
||||
|
||||
assert.Equal(t, "50", c.get("50").Id, "it returns the right bundle")
|
||||
assert.Nil(t, c.get("unknown"), "unknown bundle ist nil")
|
||||
|
||||
time.Sleep(ttl + 1)
|
||||
// roles cache has no access to evict, adding new items triggers a cleanup
|
||||
c.set("evict", nil)
|
||||
assert.Nil(t, c.get("50"), "old bundles get removed")
|
||||
}
|
||||
@@ -5,23 +5,23 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
ocisstore "github.com/owncloud/ocis/v2/ocis-pkg/store"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/store"
|
||||
settingsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0"
|
||||
settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
|
||||
"go-micro.dev/v4/store"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
)
|
||||
|
||||
const (
|
||||
cacheDatabase = "ocis-pkg"
|
||||
cacheTableName = "ocis-pkg/roles"
|
||||
cacheTableName = "roles"
|
||||
cacheTTL = time.Hour
|
||||
)
|
||||
|
||||
// Manager manages a cache of roles by fetching unknown roles from the settings.RoleService.
|
||||
type Manager struct {
|
||||
logger log.Logger
|
||||
cache store.Store
|
||||
roleCache microstore.Store
|
||||
roleService settingssvc.RoleService
|
||||
}
|
||||
|
||||
@@ -29,9 +29,9 @@ type Manager struct {
|
||||
func NewManager(o ...Option) Manager {
|
||||
opts := newOptions(o...)
|
||||
|
||||
nStore := ocisstore.Create(opts.storeOptions...)
|
||||
nStore := store.Create(opts.storeOptions...)
|
||||
return Manager{
|
||||
cache: nStore,
|
||||
roleCache: nStore,
|
||||
roleService: opts.roleService,
|
||||
}
|
||||
}
|
||||
@@ -42,7 +42,7 @@ func (m *Manager) List(ctx context.Context, roleIDs []string) []*settingsmsg.Bun
|
||||
result := make([]*settingsmsg.Bundle, 0)
|
||||
lookup := make([]string, 0)
|
||||
for _, roleID := range roleIDs {
|
||||
if records, err := m.cache.Read(roleID, store.ReadFrom(cacheDatabase, cacheTableName)); err != nil {
|
||||
if records, err := m.roleCache.Read(roleID, microstore.ReadFrom(cacheDatabase, cacheTableName)); err != nil {
|
||||
lookup = append(lookup, roleID)
|
||||
} else {
|
||||
role := &settingsmsg.Bundle{}
|
||||
@@ -77,15 +77,15 @@ func (m *Manager) List(ctx context.Context, roleIDs []string) []*settingsmsg.Bun
|
||||
}
|
||||
for _, role := range res.Bundles {
|
||||
jsonbytes, _ := protojson.Marshal(role)
|
||||
record := &store.Record{
|
||||
record := µstore.Record{
|
||||
Key: role.Id,
|
||||
Value: jsonbytes,
|
||||
Expiry: cacheTTL,
|
||||
}
|
||||
err := m.cache.Write(
|
||||
err := m.roleCache.Write(
|
||||
record,
|
||||
store.WriteTo(cacheDatabase, cacheTableName),
|
||||
store.WriteTTL(cacheTTL),
|
||||
microstore.WriteTo(cacheDatabase, cacheTableName),
|
||||
microstore.WriteTTL(cacheTTL),
|
||||
)
|
||||
if err != nil {
|
||||
m.logger.Debug().Err(err).Msg("failed to cache roles")
|
||||
|
||||
@@ -2,13 +2,13 @@ package roles
|
||||
|
||||
import (
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
ocisstore "github.com/owncloud/ocis/v2/ocis-pkg/store"
|
||||
settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
|
||||
"go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
// Options are all the possible options.
|
||||
type Options struct {
|
||||
storeOptions []ocisstore.Option
|
||||
storeOptions []store.Option
|
||||
logger log.Logger
|
||||
roleService settingssvc.RoleService
|
||||
}
|
||||
@@ -31,7 +31,7 @@ func RoleService(rs settingssvc.RoleService) Option {
|
||||
}
|
||||
|
||||
// StoreOptions are the options for the store
|
||||
func StoreOptions(storeOpts []ocisstore.Option) Option {
|
||||
func StoreOptions(storeOpts []store.Option) Option {
|
||||
return func(o *Options) {
|
||||
o.storeOptions = storeOpts
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package shared
|
||||
|
||||
import "time"
|
||||
|
||||
// EnvBinding represents a direct binding from an env variable to a go kind. Along with gookit/config, its primal goal
|
||||
// is to unpack environment variables into a Go value. We do so with reflection, and this data structure is just a step
|
||||
// in between.
|
||||
@@ -53,10 +55,13 @@ type HTTPServiceTLS struct {
|
||||
Key string `yaml:"key" env:"OCIS_HTTP_TLS_KEY" desc:"Path/File name for the TLS certificate key (in PEM format) for the server certificate to use for the http services."`
|
||||
}
|
||||
|
||||
type CacheStore struct {
|
||||
Type string `yaml:"type" env:"OCIS_CACHE_STORE_TYPE" desc:"The type of the cache store. Valid options are \"noop\", \"ocmem\", \"etcd\" and \"memory\""`
|
||||
Address string `yaml:"address" env:"OCIS_CACHE_STORE_ADDRESS" desc:"A comma-separated list of addresses to connect to. Only valid if the above setting is set to \"etcd\""`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_STORE_SIZE" desc:"Maximum size for the cache store. Only ocmem will use this option, in number of items per table. The rest will ignore the option and can grow indefinitely"`
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;OCIS_CACHE_STORE_TYPE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;OCIS_CACHE_STORE_ADDRESSES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"OCIS_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"OCIS_CACHE_STORE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_STORE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured."`
|
||||
}
|
||||
|
||||
// Commons holds configuration that are common to all extensions. Each extension can then decide whether
|
||||
@@ -64,7 +69,7 @@ type CacheStore struct {
|
||||
type Commons struct {
|
||||
Log *Log `yaml:"log"`
|
||||
Tracing *Tracing `yaml:"tracing"`
|
||||
CacheStore *CacheStore `yaml:"cache_store"`
|
||||
Cache *Cache `yaml:"cache"`
|
||||
GRPCClientTLS *GRPCClientTLS `yaml:"grpc_client_tls"`
|
||||
GRPCServiceTLS *GRPCServiceTLS `yaml:"grpc_service_tls"`
|
||||
HTTPServiceTLS HTTPServiceTLS `yaml:"http_service_tls"`
|
||||
|
||||
118
ocis-pkg/store/cache.go
Normal file
118
ocis-pkg/store/cache.go
Normal file
@@ -0,0 +1,118 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
natsjs "github.com/go-micro/plugins/v4/store/nats-js"
|
||||
"github.com/go-micro/plugins/v4/store/redis"
|
||||
redisopts "github.com/go-redis/redis/v8"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/store/etcd"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/store/memory"
|
||||
"go-micro.dev/v4/logger"
|
||||
"go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
var ocMemStore *store.Store
|
||||
|
||||
const (
|
||||
TypeMemory = "memory"
|
||||
TypeNoop = "noop"
|
||||
TypeEtcd = "etcd"
|
||||
TypeRedis = "redis"
|
||||
TypeRedisSentinel = "redis-sentinel"
|
||||
TypeOCMem = "ocmem"
|
||||
TypeNatsJS = "nats-js"
|
||||
)
|
||||
|
||||
// Create returns a configured key-value micro store
|
||||
//
|
||||
// Each microservice (or whatever piece is using the store) should use the
|
||||
// options available in the interface's operations to choose the right database
|
||||
// and table to prevent collisions with other microservices.
|
||||
// Recommended approach is to use "services" or "ocis-pkg" for the database,
|
||||
// and "services/<service-name>/" or "ocis-pkg/<pkg>/" for the package name.
|
||||
func Create(opts ...store.Option) store.Store {
|
||||
options := &store.Options{
|
||||
Context: context.Background(),
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(options)
|
||||
}
|
||||
|
||||
storeType, _ := options.Context.Value(typeContextKey{}).(string)
|
||||
|
||||
switch storeType {
|
||||
case TypeNoop:
|
||||
return store.NewNoopStore(opts...)
|
||||
case TypeEtcd:
|
||||
return etcd.NewEtcdStore(opts...)
|
||||
case TypeRedis:
|
||||
// FIXME redis plugin does not support redis cluster or ring -> needs upstream patch or our implementation
|
||||
return redis.NewStore(opts...)
|
||||
case TypeRedisSentinel:
|
||||
redisMaster := ""
|
||||
redisNodes := []string{}
|
||||
for _, node := range options.Nodes {
|
||||
parts := strings.SplitN(node, "/", 2)
|
||||
if len(parts) != 2 {
|
||||
return nil
|
||||
}
|
||||
// the first node is used to retrieve the redis master
|
||||
redisNodes = append(redisNodes, parts[0])
|
||||
if redisMaster == "" {
|
||||
redisMaster = parts[1]
|
||||
}
|
||||
}
|
||||
return redis.NewStore(
|
||||
store.Database(options.Database),
|
||||
store.Table(options.Table),
|
||||
store.Nodes(redisNodes...),
|
||||
redis.WithRedisOptions(redisopts.UniversalOptions{
|
||||
MasterName: redisMaster,
|
||||
}),
|
||||
)
|
||||
case TypeOCMem:
|
||||
if ocMemStore == nil {
|
||||
var memStore store.Store
|
||||
|
||||
sizeNum, _ := options.Context.Value(sizeContextKey{}).(int)
|
||||
if sizeNum <= 0 {
|
||||
memStore = memory.NewMultiMemStore()
|
||||
} else {
|
||||
memStore = memory.NewMultiMemStore(
|
||||
store.WithContext(
|
||||
memory.NewContext(
|
||||
context.Background(),
|
||||
map[string]interface{}{
|
||||
"maxCap": sizeNum,
|
||||
},
|
||||
)),
|
||||
)
|
||||
}
|
||||
ocMemStore = &memStore
|
||||
}
|
||||
return *ocMemStore
|
||||
case TypeNatsJS:
|
||||
ttl, _ := options.Context.Value(ttlContextKey{}).(time.Duration)
|
||||
// TODO nats needs a DefaultTTL option as it does not support per Write TTL ...
|
||||
// FIXME nats has restrictions on the key, we cannot use slashes AFAICT
|
||||
// host, port, clusterid
|
||||
return natsjs.NewStore(
|
||||
append(opts,
|
||||
natsjs.NatsOptions(nats.Options{Name: "TODO"}),
|
||||
natsjs.DefaultTTL(ttl))...,
|
||||
) // TODO test with ocis nats
|
||||
case TypeMemory, "mem", "": // allow existing short form and use as default
|
||||
return store.NewMemoryStore(opts...)
|
||||
default:
|
||||
// try to log an error
|
||||
if options.Logger == nil {
|
||||
options.Logger = logger.DefaultLogger
|
||||
}
|
||||
options.Logger.Logf(logger.ErrorLevel, "unknown store type: '%s', falling back to memory", storeType)
|
||||
return store.NewMemoryStore(opts...)
|
||||
}
|
||||
}
|
||||
@@ -1,50 +1,59 @@
|
||||
package store
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
// Option provides an option to configure the store
|
||||
type Option func(*Options)
|
||||
"go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
// Type defines the type of the store
|
||||
func Type(typ string) Option {
|
||||
return func(o *Options) {
|
||||
o.Type = typ
|
||||
type typeContextKey struct{}
|
||||
|
||||
// Store determines the implementation:
|
||||
// - "memory", for a in-memory implementation, which is also the default if noone matches
|
||||
// - "noop", for a noop store (it does nothing)
|
||||
// - "etcd", for etcd
|
||||
// - "nats-js" for nats-js, needs to have TTL configured at creation
|
||||
// - "redis", for redis
|
||||
// - "redis-sentinel", for redis-sentinel
|
||||
// - "ocmem", custom in-memory implementation, with fixed size and optimized prefix
|
||||
// and suffix search
|
||||
func Store(val string) store.Option {
|
||||
return func(o *store.Options) {
|
||||
if o.Context == nil {
|
||||
o.Context = context.Background()
|
||||
}
|
||||
|
||||
o.Context = context.WithValue(o.Context, typeContextKey{}, val)
|
||||
}
|
||||
}
|
||||
|
||||
// Addresses defines the addresses where the store can be reached
|
||||
func Addresses(addrs ...string) Option {
|
||||
return func(o *Options) {
|
||||
o.Addresses = addrs
|
||||
type sizeContextKey struct{}
|
||||
|
||||
// Size configures the maximum capacity of the cache for the "ocmem" implementation,
|
||||
// in number of items that the cache can hold per table.
|
||||
// You can use 5000 to make the cache hold up to 5000 elements.
|
||||
// The parameter only affects to the "ocmem" implementation, the rest will ignore it.
|
||||
// If an invalid value is used, the default of 512 will be used instead.
|
||||
func Size(val int) store.Option {
|
||||
return func(o *store.Options) {
|
||||
if o.Context == nil {
|
||||
o.Context = context.Background()
|
||||
}
|
||||
|
||||
o.Context = context.WithValue(o.Context, sizeContextKey{}, val)
|
||||
}
|
||||
}
|
||||
|
||||
// Database defines the Database the store should use
|
||||
func Database(db string) Option {
|
||||
return func(o *Options) {
|
||||
o.Database = db
|
||||
}
|
||||
}
|
||||
type ttlContextKey struct{}
|
||||
|
||||
// Table defines the table the store should use
|
||||
func Table(t string) Option {
|
||||
return func(o *Options) {
|
||||
o.Table = t
|
||||
}
|
||||
}
|
||||
// TTL is the time to live for documents stored in the store
|
||||
func TTL(val time.Duration) store.Option {
|
||||
return func(o *store.Options) {
|
||||
if o.Context == nil {
|
||||
o.Context = context.Background()
|
||||
}
|
||||
|
||||
// Size defines the maximum capacity of the store.
|
||||
// Only applicable when using "ocmem" store
|
||||
func Size(s int) Option {
|
||||
return func(o *Options) {
|
||||
o.Size = s
|
||||
}
|
||||
}
|
||||
|
||||
// TTL defines the time to life for elements in the store.
|
||||
// Only applicable when using "natsjs" store
|
||||
func TTL(t time.Duration) Option {
|
||||
return func(o *Options) {
|
||||
o.TTL = t
|
||||
o.Context = context.WithValue(o.Context, ttlContextKey{}, val)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
natsjs "github.com/go-micro/plugins/v4/store/nats-js"
|
||||
"github.com/go-micro/plugins/v4/store/redis"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/store/etcd"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/store/memory"
|
||||
"go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
var ocMemStore *store.Store
|
||||
|
||||
// Options are the options to configure the store
|
||||
type Options struct {
|
||||
// Type determines the implementation:
|
||||
// * "noop", for a noop store (it does nothing)
|
||||
// * "etcd", for etcd
|
||||
// * "ocmem", custom in-memory implementation, with fixed size and optimized prefix
|
||||
// and suffix search
|
||||
// * "memory", for a in-memory implementation, which is the default if noone matches
|
||||
Type string
|
||||
|
||||
// Address is a list of nodes that the store will use.
|
||||
Addresses []string
|
||||
|
||||
// Size configures the maximum capacity of the cache for
|
||||
// the "ocmem" implementation, in number of items that the cache can hold per table.
|
||||
// You can use 5000 to make the cache hold up to 5000 elements.
|
||||
// The parameter only affects to the "ocmem" implementation, the rest will ignore it.
|
||||
// If an invalid value is used, the default of 512 will be used instead.
|
||||
Size int
|
||||
|
||||
// Database the store should use (optional)
|
||||
Database string
|
||||
|
||||
// Table the store should use (optional)
|
||||
Table string
|
||||
|
||||
// TTL is the time to life for documents stored in the store
|
||||
TTL time.Duration
|
||||
}
|
||||
|
||||
// Create returns a configured key-value store
|
||||
//
|
||||
// Each microservice (or whatever piece is using the store) should use the
|
||||
// options available in the interface's operations to choose the right database
|
||||
// and table to prevent collisions with other microservices.
|
||||
// Recommended approach is to use "services" or "ocis-pkg" for the database,
|
||||
// and "services/<service-name>/" or "ocis-pkg/<pkg>/" for the package name.
|
||||
func Create(opts ...Option) store.Store {
|
||||
options := &Options{}
|
||||
for _, o := range opts {
|
||||
o(options)
|
||||
}
|
||||
|
||||
storeopts := storeOptions(options)
|
||||
|
||||
switch options.Type {
|
||||
default:
|
||||
// TODO: better to error in default case?
|
||||
fallthrough
|
||||
case "mem":
|
||||
return store.NewMemoryStore(storeopts...)
|
||||
case "noop":
|
||||
return store.NewNoopStore(storeopts...)
|
||||
case "etcd":
|
||||
return etcd.NewEtcdStore(storeopts...)
|
||||
case "redis":
|
||||
// FIXME redis plugin does not support redis cluster, sentinel or ring -> needs upstream patch or our implementation
|
||||
return redis.NewStore(storeopts...)
|
||||
case "ocmem":
|
||||
if ocMemStore == nil {
|
||||
var memStore store.Store
|
||||
|
||||
sizeNum := options.Size
|
||||
if sizeNum <= 0 {
|
||||
memStore = memory.NewMultiMemStore()
|
||||
} else {
|
||||
memStore = memory.NewMultiMemStore(
|
||||
store.WithContext(
|
||||
memory.NewContext(
|
||||
context.Background(),
|
||||
map[string]interface{}{
|
||||
"maxCap": sizeNum,
|
||||
},
|
||||
)),
|
||||
)
|
||||
}
|
||||
ocMemStore = &memStore
|
||||
}
|
||||
return *ocMemStore
|
||||
case "nats-js":
|
||||
// TODO nats needs a DefaultTTL option as it does not support per Write TTL ...
|
||||
// FIXME nats has restrictions on the key, we cannot use slashes AFAICT
|
||||
// host, port, clusterid
|
||||
return natsjs.NewStore(
|
||||
append(storeopts,
|
||||
natsjs.NatsOptions(nats.Options{Name: "TODO"}),
|
||||
natsjs.DefaultTTL(options.TTL),
|
||||
)...,
|
||||
) // TODO test with ocis nats
|
||||
}
|
||||
}
|
||||
|
||||
func storeOptions(o *Options) []store.Option {
|
||||
var opts []store.Option
|
||||
|
||||
if o.Addresses != nil {
|
||||
opts = append(opts, store.Nodes(o.Addresses...))
|
||||
}
|
||||
|
||||
if o.Database != "" {
|
||||
opts = append(opts, store.Database(o.Database))
|
||||
|
||||
}
|
||||
|
||||
if o.Table != "" {
|
||||
opts = append(opts, store.Table(o.Table))
|
||||
|
||||
}
|
||||
|
||||
return opts
|
||||
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package command
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/events/stream"
|
||||
"github.com/oklog/run"
|
||||
@@ -17,6 +16,7 @@ import (
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/metrics"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/server/grpc"
|
||||
"github.com/urfave/cli/v2"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
// Server is the entrypoint for the server command.
|
||||
@@ -56,11 +56,12 @@ func Server(cfg *config.Config) *cli.Command {
|
||||
}
|
||||
|
||||
st := store.Create(
|
||||
store.Type(cfg.Store.Type),
|
||||
store.Addresses(strings.Split(cfg.Store.Addresses, ",")...),
|
||||
store.Database(cfg.Store.Database),
|
||||
store.Table(cfg.Store.Table),
|
||||
store.Store(cfg.Store.Store),
|
||||
store.TTL(cfg.Store.RecordExpiry),
|
||||
store.Size(cfg.Store.Size),
|
||||
microstore.Nodes(cfg.Store.Nodes...),
|
||||
microstore.Database(cfg.Store.Database),
|
||||
microstore.Table(cfg.Store.Table),
|
||||
)
|
||||
|
||||
service := grpc.NewService(
|
||||
@@ -72,7 +73,7 @@ func Server(cfg *config.Config) *cli.Command {
|
||||
grpc.Address(cfg.GRPC.Addr),
|
||||
grpc.Metrics(metrics),
|
||||
grpc.Consumer(consumer),
|
||||
grpc.Store(st),
|
||||
grpc.Persistence(st),
|
||||
)
|
||||
|
||||
gr.Add(service.Run, func(err error) {
|
||||
|
||||
@@ -34,11 +34,11 @@ type GRPCConfig struct {
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Type string `yaml:"type" env:"EVENTHISTORY_STORE_TYPE" desc:"The type of the eventhistory store. Supported values are: 'mem', 'ocmem', 'etcd', 'redis', 'nats-js', 'noop'. See the text description for details."`
|
||||
Addresses string `yaml:"addresses" env:"EVENTHISTORY_STORE_ADDRESSES" desc:"A comma separated list of addresses to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"EVENTHISTORY_STORE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when 'in-memory' stores are configured."`
|
||||
Table string `yaml:"table" env:"EVENTHISTORY_STORE_TABLE" desc:"(optional) The database table the store should use. This has no effect when 'in-memory' stores are configured."`
|
||||
RecordExpiry time.Duration `yaml:"record_expiry" env:"EVENTHISTORY_RECORD_EXPIRY" desc:"Time to life for events in the store. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '336h' (2 weeks)."`
|
||||
Store string `yaml:"store" env:"OCIS_PERSISTENT_STORE;EVENTHISTORY_STORE;OCIS_PERSISTENT_STORE_TYPE;EVENTHISTORY_STORE_TYPE" desc:"The type of the eventhistory store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_PERSISTENT_STORE_NODES;EVENTHISTORY_STORE_ADDRESSES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"EVENTHISTORY_STORE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"EVENTHISTORY_STORE_TABLE" desc:"The database table the store should use."`
|
||||
RecordExpiry time.Duration `yaml:"record_expiry" env:"EVENTHISTORY_RECORD_EXPIRY" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '336h' (2 weeks)."`
|
||||
Size int `yaml:"size" env:"EVENTHISTORY_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured. Defaults to 512."`
|
||||
}
|
||||
|
||||
|
||||
@@ -27,7 +27,9 @@ func DefaultConfig() *config.Config {
|
||||
EnableTLS: false,
|
||||
},
|
||||
Store: config.Store{
|
||||
Type: "mem",
|
||||
Store: "memory",
|
||||
Database: "eventhistory",
|
||||
Table: "events",
|
||||
RecordExpiry: 336 * time.Hour,
|
||||
},
|
||||
GRPC: config.GRPCConfig{
|
||||
|
||||
@@ -16,16 +16,16 @@ type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Name string
|
||||
Address string
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Metrics *metrics.Metrics
|
||||
Namespace string
|
||||
Flags []cli.Flag
|
||||
Store store.Store
|
||||
Consumer events.Consumer
|
||||
Name string
|
||||
Address string
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Metrics *metrics.Metrics
|
||||
Namespace string
|
||||
Flags []cli.Flag
|
||||
Persistence store.Store
|
||||
Consumer events.Consumer
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
@@ -95,10 +95,10 @@ func Flags(flags []cli.Flag) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// Store provides a function to configure the store
|
||||
func Store(store store.Store) Option {
|
||||
// Persistence provides a function to configure the store
|
||||
func Persistence(store store.Store) Option {
|
||||
return func(o *Options) {
|
||||
o.Store = store
|
||||
o.Persistence = store
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ func NewService(opts ...Option) grpc.Service {
|
||||
return grpc.Service{}
|
||||
}
|
||||
|
||||
eh, err := svc.NewEventHistoryService(options.Config, options.Consumer, options.Store, options.Logger)
|
||||
eh, err := svc.NewEventHistoryService(options.Config, options.Consumer, options.Persistence, options.Logger)
|
||||
if err != nil {
|
||||
options.Logger.Fatal().Err(err).Msg("Error creating event history service")
|
||||
return grpc.Service{}
|
||||
|
||||
@@ -88,10 +88,10 @@ type StorageRegistry struct {
|
||||
|
||||
// Cache holds cache config
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;GATEWAY_CACHE_STORE_TYPE;GATEWAY_CACHE_STORE" desc:"Store implementation for the cache. Valid values are \"memory\" (default), \"redis\", and \"etcd\"."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;GATEWAY_CACHE_STORE_ADDRESS;GATEWAY_CACHE_NODES" desc:"Node addresses to use for the cache store."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;GATEWAY_CACHE_STORE;OCIS_CACHE_STORE_TYPE;GATEWAY_CACHE_STORE_TYPE" desc:"Store implementation for the cache. Valid values are \"memory\" (default), \"redis\", and \"etcd\"."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CACHE_STORE_NODES;OCIS_CACHE_STORE_ADDRESS;GATEWAY_CACHE_STORE_ADDRESS;GATEWAY_CACHE_NODES" desc:"Nodes to use for the cache store."`
|
||||
Database string `yaml:"database" env:"GATEWAY_CACHE_DATABASE" desc:"Database name of the cache."`
|
||||
StatCacheTTL int `yaml:"stat_cache_ttl" env:"GATEWAY_STAT_CACHE_TTL" desc:"Max TTL in seconds for the gateway's stat cache."`
|
||||
ProviderCacheTTL int `yaml:"provider_cache_ttl" env:"GATEWAY_PROVIDER_CACHE_TTL" desc:"Max TTL in seconds for the gateway's provider cache."`
|
||||
CreateHomeCacheTTL int `yaml:"create_home_cache_ttl" env:"GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Max TTL in seconds for the gateway's create home cache."`
|
||||
StatCacheTTL int `yaml:"stat_cache_ttl" env:"OCIS_CACHE_STORE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Max TTL in seconds for the gateway's stat cache."`
|
||||
ProviderCacheTTL int `yaml:"provider_cache_ttl" env:"OCIS_CACHE_STORE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Max TTL in seconds for the gateway's provider cache."`
|
||||
CreateHomeCacheTTL int `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_STORE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Max TTL in seconds for the gateway's create home cache."`
|
||||
}
|
||||
|
||||
13
services/graph/pkg/config/cache.go
Normal file
13
services/graph/pkg/config/cache.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package config
|
||||
|
||||
import "time"
|
||||
|
||||
// Cache defines the available configuration for a cache store
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;GRAPH_CACHE_STORE;OCIS_CACHE_STORE_TYPE;GRAPH_CACHE_STORE_TYPE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;GRAPH_CACHE_STORE_NODES;OCIS_CACHE_STORE_ADDRESSES;GRAPH_CACHE_STORE_ADDRESSES" desc:"A comma-separated list of nodes to connect to. This has no effect when 'in-memory' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"GRAPH_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"GRAPH_CACHE_STORE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_STORE_TTL;GRAPH_CACHE_STORE_TTL" desc:"Time to live for cache records in the graph. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '336h' (2 weeks)."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_STORE_SIZE;GRAPH_CACHE_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured. Defaults to 512."`
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
package config
|
||||
|
||||
// CacheStore defines the available configuration for the cache store
|
||||
type CacheStore struct {
|
||||
Type string `yaml:"type" env:"OCIS_CACHE_STORE_TYPE;GRAPH_CACHE_STORE_TYPE" desc:"The type of the cache store. Valid options are \"noop\", \"ocmem\", \"etcd\" and \"memory\""`
|
||||
Address string `yaml:"address" env:"OCIS_CACHE_STORE_ADDRESS;GRAPH_CACHE_STORE_ADDRESS" desc:"A comma-separated list of addresses to connect to. Only valid if the above setting is set to \"etcd\""`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_STORE_SIZE;GRAPH_CACHE_STORE_SIZE" desc:"Maximum number of items per table in the ocmem cache store. Other cache stores will ignore the option and can grow indefinitely."`
|
||||
}
|
||||
@@ -12,10 +12,10 @@ type Config struct {
|
||||
|
||||
Service Service `yaml:"-"`
|
||||
|
||||
Tracing *Tracing `yaml:"tracing"`
|
||||
Log *Log `yaml:"log"`
|
||||
CacheStore *CacheStore `yaml:"cache_store"`
|
||||
Debug Debug `yaml:"debug"`
|
||||
Tracing *Tracing `yaml:"tracing"`
|
||||
Log *Log `yaml:"log"`
|
||||
Cache *Cache `yaml:"cache"`
|
||||
Debug Debug `yaml:"debug"`
|
||||
|
||||
HTTP HTTP `yaml:"http"`
|
||||
|
||||
|
||||
@@ -85,6 +85,12 @@ func DefaultConfig() *config.Config {
|
||||
EducationResourcesEnabled: false,
|
||||
},
|
||||
},
|
||||
Cache: &config.Cache{
|
||||
Store: "memory",
|
||||
Database: "graph",
|
||||
Table: "roles",
|
||||
TTL: time.Hour * 336,
|
||||
},
|
||||
Events: config.Events{
|
||||
Endpoint: "127.0.0.1:9233",
|
||||
Cluster: "ocis-cluster",
|
||||
@@ -118,14 +124,14 @@ func EnsureDefaults(cfg *config.Config) {
|
||||
cfg.Tracing = &config.Tracing{}
|
||||
}
|
||||
|
||||
if cfg.CacheStore == nil && cfg.Commons != nil && cfg.Commons.CacheStore != nil {
|
||||
cfg.CacheStore = &config.CacheStore{
|
||||
Type: cfg.Commons.CacheStore.Type,
|
||||
Address: cfg.Commons.CacheStore.Address,
|
||||
Size: cfg.Commons.CacheStore.Size,
|
||||
if cfg.Cache == nil && cfg.Commons != nil && cfg.Commons.Cache != nil {
|
||||
cfg.Cache = &config.Cache{
|
||||
Store: cfg.Commons.Cache.Store,
|
||||
Nodes: cfg.Commons.Cache.Nodes,
|
||||
Size: cfg.Commons.Cache.Size,
|
||||
}
|
||||
} else if cfg.CacheStore == nil {
|
||||
cfg.CacheStore = &config.CacheStore{}
|
||||
} else if cfg.Cache == nil {
|
||||
cfg.Cache = &config.Cache{}
|
||||
}
|
||||
|
||||
if cfg.TokenManager == nil && cfg.Commons != nil && cfg.Commons.TokenManager != nil {
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@@ -24,6 +23,7 @@ import (
|
||||
"github.com/owncloud/ocis/v2/services/graph/pkg/identity"
|
||||
"github.com/owncloud/ocis/v2/services/graph/pkg/identity/ldap"
|
||||
graphm "github.com/owncloud/ocis/v2/services/graph/pkg/middleware"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -159,10 +159,13 @@ func NewService(opts ...Option) (Graph, error) {
|
||||
|
||||
roleManager := options.RoleManager
|
||||
if roleManager == nil {
|
||||
storeOptions := []store.Option{
|
||||
store.Type(options.Config.CacheStore.Type),
|
||||
store.Addresses(strings.Split(options.Config.CacheStore.Address, ",")...),
|
||||
store.Size(options.Config.CacheStore.Size),
|
||||
storeOptions := []microstore.Option{
|
||||
store.Store(options.Config.Cache.Store),
|
||||
store.TTL(options.Config.Cache.TTL),
|
||||
store.Size(options.Config.Cache.Size),
|
||||
microstore.Nodes(options.Config.Cache.Nodes...),
|
||||
microstore.Database(options.Config.Cache.Database),
|
||||
microstore.Table(options.Config.Cache.Table),
|
||||
}
|
||||
m := roles.NewManager(
|
||||
roles.StoreOptions(storeOptions),
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
package config
|
||||
|
||||
// CacheStore defines the available configuration for the cache store
|
||||
type CacheStore struct {
|
||||
Type string `yaml:"type" env:"OCIS_CACHE_STORE_TYPE;OCS_CACHE_STORE_TYPE" desc:"The type of the cache store. Valid options are \"noop\", \"ocmem\", \"etcd\" and \"memory\""`
|
||||
Address string `yaml:"address" env:"OCIS_CACHE_STORE_ADDRESS;OCS_CACHE_STORE_ADDRESS" desc:"A comma-separated list of addresses to connect to. Only valid if the above setting is set to \"etcd\""`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_STORE_SIZE;OCS_CACHE_STORE_SIZE" desc:"Maximum number of items per table in the ocmem cache store. Other cache stores will ignore the option and can grow indefinitely."`
|
||||
import "time"
|
||||
|
||||
// Cache defines the available configuration for the cache store
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;OCS_CACHE_STORE;OCIS_CACHE_STORE_TYPE;OCS_CACHE_STORE_TYPE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;OCS_CACHE_STORE_NODES;OCIS_CACHE_STORE_ADDRESSES;OCS_CACHE_STORE_ADDRESSES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"OCS_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"OCS_CACHE_STORE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_STORE_TTL;OCS_CACHE_STORE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '336h' (2 weeks)."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_STORE_SIZE;OCS_CACHE_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured. Defaults to 512."`
|
||||
}
|
||||
|
||||
@@ -12,10 +12,10 @@ type Config struct {
|
||||
|
||||
Service Service `yaml:"-"`
|
||||
|
||||
Tracing *Tracing `yaml:"tracing"`
|
||||
Log *Log `yaml:"log"`
|
||||
CacheStore *CacheStore `yaml:"cache_store"`
|
||||
Debug Debug `yaml:"debug"`
|
||||
Tracing *Tracing `yaml:"tracing"`
|
||||
Log *Log `yaml:"log"`
|
||||
Cache *Cache `yaml:"cache"`
|
||||
Debug Debug `yaml:"debug"`
|
||||
|
||||
HTTP HTTP `yaml:"http"`
|
||||
|
||||
|
||||
@@ -65,14 +65,14 @@ func EnsureDefaults(cfg *config.Config) {
|
||||
cfg.Tracing = &config.Tracing{}
|
||||
}
|
||||
|
||||
if cfg.CacheStore == nil && cfg.Commons != nil && cfg.Commons.CacheStore != nil {
|
||||
cfg.CacheStore = &config.CacheStore{
|
||||
Type: cfg.Commons.CacheStore.Type,
|
||||
Address: cfg.Commons.CacheStore.Address,
|
||||
Size: cfg.Commons.CacheStore.Size,
|
||||
if cfg.Cache == nil && cfg.Commons != nil && cfg.Commons.Cache != nil {
|
||||
cfg.Cache = &config.Cache{
|
||||
Store: cfg.Commons.Cache.Store,
|
||||
Nodes: cfg.Commons.Cache.Nodes,
|
||||
Size: cfg.Commons.Cache.Size,
|
||||
}
|
||||
} else if cfg.CacheStore == nil {
|
||||
cfg.CacheStore = &config.CacheStore{}
|
||||
} else if cfg.Cache == nil {
|
||||
cfg.Cache = &config.Cache{}
|
||||
}
|
||||
|
||||
if cfg.TokenManager == nil && cfg.Commons != nil && cfg.Commons.TokenManager != nil {
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
pkgmiddleware "github.com/owncloud/ocis/v2/ocis-pkg/middleware"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/store"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/version"
|
||||
settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
|
||||
storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0"
|
||||
@@ -34,6 +35,7 @@ import (
|
||||
"github.com/owncloud/ocis/v2/services/proxy/pkg/user/backend"
|
||||
"github.com/owncloud/ocis/v2/services/proxy/pkg/userroles"
|
||||
"github.com/urfave/cli/v2"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
@@ -209,12 +211,25 @@ func loadMiddlewares(ctx context.Context, logger log.Logger, cfg *config.Config)
|
||||
UserProvider: userProvider,
|
||||
})
|
||||
}
|
||||
|
||||
cache := store.Create(
|
||||
store.Store(cfg.OIDC.UserinfoCache.Store),
|
||||
store.TTL(cfg.OIDC.UserinfoCache.TTL),
|
||||
store.Size(cfg.OIDC.UserinfoCache.Size),
|
||||
microstore.Nodes(cfg.OIDC.UserinfoCache.Nodes...),
|
||||
microstore.Database(cfg.OIDC.UserinfoCache.Database),
|
||||
microstore.Table(cfg.OIDC.UserinfoCache.Table),
|
||||
)
|
||||
|
||||
authenticators = append(authenticators, middleware.NewOIDCAuthenticator(
|
||||
logger,
|
||||
cfg.OIDC.UserinfoCache.TTL,
|
||||
oidcHTTPClient,
|
||||
cfg.OIDC.Issuer,
|
||||
func() (middleware.OIDCProvider, error) {
|
||||
middleware.Logger(logger),
|
||||
middleware.Cache(cache),
|
||||
middleware.DefaultAccessTokenTTL(cfg.OIDC.UserinfoCache.TTL),
|
||||
middleware.HTTPClient(oidcHTTPClient),
|
||||
middleware.OIDCIss(cfg.OIDC.Issuer),
|
||||
middleware.JWKSOptions(cfg.OIDC.JWKS),
|
||||
middleware.AccessTokenVerifyMethod(cfg.OIDC.AccessTokenVerifyMethod),
|
||||
middleware.OIDCProviderFunc(func() (middleware.OIDCProvider, error) {
|
||||
// Initialize a provider by specifying the issuer URL.
|
||||
// it will fetch the keys from the issuer using the .well-known
|
||||
// endpoint
|
||||
@@ -222,9 +237,7 @@ func loadMiddlewares(ctx context.Context, logger log.Logger, cfg *config.Config)
|
||||
context.WithValue(ctx, oauth2.HTTPClient, oidcHTTPClient),
|
||||
cfg.OIDC.Issuer,
|
||||
)
|
||||
},
|
||||
cfg.OIDC.JWKS,
|
||||
cfg.OIDC.AccessTokenVerifyMethod,
|
||||
}),
|
||||
))
|
||||
authenticators = append(authenticators, middleware.PublicShareAuthenticator{
|
||||
Logger: logger,
|
||||
|
||||
@@ -2,6 +2,7 @@ package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
|
||||
)
|
||||
@@ -101,12 +102,12 @@ const (
|
||||
// OIDC is the config for the OpenID-Connect middleware. If set the proxy will try to authenticate every request
|
||||
// with the configured oidc-provider
|
||||
type OIDC struct {
|
||||
Issuer string `yaml:"issuer" env:"OCIS_URL;OCIS_OIDC_ISSUER;PROXY_OIDC_ISSUER" desc:"URL of the OIDC issuer. It defaults to URL of the builtin IDP."`
|
||||
Insecure bool `yaml:"insecure" env:"OCIS_INSECURE;PROXY_OIDC_INSECURE" desc:"Disable TLS certificate validation for connections to the IDP. Note that this is not recommended for production environments."`
|
||||
AccessTokenVerifyMethod string `yaml:"access_token_verify_method" env:"PROXY_OIDC_ACCESS_TOKEN_VERIFY_METHOD" desc:"Sets how OIDC access tokens should be verified. Possible values are 'none' and 'jwt'. When using 'none', no special validation apart from using it for accessing the IPD's userinfo endpoint will be done. When using 'jwt', it tries to parse the access token as a jwt token and verifies the signature using the keys published on the IDP's 'jwks_uri'."`
|
||||
UserinfoCache UserinfoCache `yaml:"user_info_cache"`
|
||||
JWKS JWKS `yaml:"jwks"`
|
||||
RewriteWellKnown bool `yaml:"rewrite_well_known" env:"PROXY_OIDC_REWRITE_WELLKNOWN" desc:"Enables rewriting the /.well-known/openid-configuration to the configured OIDC issuer. Needed by the Desktop Client, Android Client and iOS Client to discover the OIDC provider."`
|
||||
Issuer string `yaml:"issuer" env:"OCIS_URL;OCIS_OIDC_ISSUER;PROXY_OIDC_ISSUER" desc:"URL of the OIDC issuer. It defaults to URL of the builtin IDP."`
|
||||
Insecure bool `yaml:"insecure" env:"OCIS_INSECURE;PROXY_OIDC_INSECURE" desc:"Disable TLS certificate validation for connections to the IDP. Note that this is not recommended for production environments."`
|
||||
AccessTokenVerifyMethod string `yaml:"access_token_verify_method" env:"PROXY_OIDC_ACCESS_TOKEN_VERIFY_METHOD" desc:"Sets how OIDC access tokens should be verified. Possible values are 'none' and 'jwt'. When using 'none', no special validation apart from using it for accessing the IPD's userinfo endpoint will be done. When using 'jwt', it tries to parse the access token as a jwt token and verifies the signature using the keys published on the IDP's 'jwks_uri'."`
|
||||
UserinfoCache *Cache `yaml:"user_info_cache"`
|
||||
JWKS JWKS `yaml:"jwks"`
|
||||
RewriteWellKnown bool `yaml:"rewrite_well_known" env:"PROXY_OIDC_REWRITE_WELLKNOWN" desc:"Enables rewriting the /.well-known/openid-configuration to the configured OIDC issuer. Needed by the Desktop Client, Android Client and iOS Client to discover the OIDC provider."`
|
||||
}
|
||||
|
||||
type JWKS struct {
|
||||
@@ -116,10 +117,14 @@ type JWKS struct {
|
||||
RefreshUnknownKID bool `yaml:"refresh_unknown_kid" env:"PROXY_OIDC_JWKS_REFRESH_UNKNOWN_KID" desc:"If set to 'true', the JWKS refresh request will occur every time an unknown KEY ID (KID) is seen. Always set a 'refresh_limit' when enabling this."`
|
||||
}
|
||||
|
||||
// UserinfoCache is a TTL cache configuration.
|
||||
type UserinfoCache struct {
|
||||
Size int `yaml:"size" env:"PROXY_OIDC_USERINFO_CACHE_SIZE" desc:"Cache size for OIDC user info."`
|
||||
TTL int `yaml:"ttl" env:"PROXY_OIDC_USERINFO_CACHE_TTL" desc:"Max TTL in seconds for the OIDC user info cache."`
|
||||
// Cache is a TTL cache configuration.
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;PROXY_OIDC_USERINFO_CACHE_STORE;OCIS_CACHE_STORE_TYPE;PROXY_OIDC_USERINFO_CACHE_TYPE" desc:"The type of the userinfo cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;PROXY_OIDC_USERINFO_CACHE_NODES;OCIS_CACHE_STORE_ADDRESSES;PROXY_OIDC_USERINFO_CACHE_ADDRESSES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"PROXY_OIDC_USERINFO_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"PROXY_OIDC_USERINFO_CACHE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_STORE_TTL;PROXY_OIDC_USERINFO_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '10s' (10 seconds)."`
|
||||
Size int `yaml:"size" env:"OCIS_CACHE_STORE_SIZE;PROXY_OIDC_USERINFO_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512."`
|
||||
}
|
||||
|
||||
// RoleAssignment contains the configuration for how to assign roles to users during login
|
||||
|
||||
@@ -3,6 +3,7 @@ package defaults
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/config/defaults"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
|
||||
@@ -40,9 +41,11 @@ func DefaultConfig() *config.Config {
|
||||
Issuer: "https://localhost:9200",
|
||||
|
||||
AccessTokenVerifyMethod: config.AccessTokenVerificationJWT,
|
||||
UserinfoCache: config.UserinfoCache{
|
||||
Size: 1024,
|
||||
TTL: 10,
|
||||
UserinfoCache: &config.Cache{
|
||||
Store: "memory",
|
||||
Database: "proxy",
|
||||
Table: "userinfo",
|
||||
TTL: time.Second * 10,
|
||||
},
|
||||
JWKS: config.JWKS{
|
||||
RefreshInterval: 60, // minutes
|
||||
@@ -254,6 +257,16 @@ func EnsureDefaults(cfg *config.Config) {
|
||||
cfg.Tracing = &config.Tracing{}
|
||||
}
|
||||
|
||||
if cfg.OIDC.UserinfoCache == nil && cfg.Commons != nil && cfg.Commons.Cache != nil {
|
||||
cfg.OIDC.UserinfoCache = &config.Cache{
|
||||
Store: cfg.Commons.Cache.Store,
|
||||
Nodes: cfg.Commons.Cache.Nodes,
|
||||
Size: cfg.Commons.Cache.Size,
|
||||
}
|
||||
} else if cfg.OIDC.UserinfoCache == nil {
|
||||
cfg.OIDC.UserinfoCache = &config.Cache{}
|
||||
}
|
||||
|
||||
if cfg.TokenManager == nil && cfg.Commons != nil && cfg.Commons.TokenManager != nil {
|
||||
cfg.TokenManager = &config.TokenManager{
|
||||
JWTSecret: cfg.Commons.TokenManager.JWTSecret,
|
||||
|
||||
@@ -2,19 +2,23 @@ package middleware
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/oidc"
|
||||
"github.com/owncloud/ocis/v2/services/proxy/pkg/config"
|
||||
|
||||
"github.com/MicahParks/keyfunc"
|
||||
gOidc "github.com/coreos/go-oidc/v3/oidc"
|
||||
"github.com/golang-jwt/jwt/v4"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/oidc"
|
||||
osync "github.com/owncloud/ocis/v2/ocis-pkg/sync"
|
||||
"github.com/owncloud/ocis/v2/services/proxy/pkg/config"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/shamaton/msgpack/v2"
|
||||
store "go-micro.dev/v4/store"
|
||||
"golang.org/x/crypto/sha3"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
@@ -29,18 +33,18 @@ type OIDCProvider interface {
|
||||
}
|
||||
|
||||
// NewOIDCAuthenticator returns a ready to use authenticator which can handle OIDC authentication.
|
||||
func NewOIDCAuthenticator(logger log.Logger, tokenCacheTTL int, oidcHTTPClient *http.Client, oidcIss string, providerFunc func() (OIDCProvider, error),
|
||||
jwksOptions config.JWKS, accessTokenVerifyMethod string) *OIDCAuthenticator {
|
||||
tokenCache := osync.NewCache(tokenCacheTTL)
|
||||
func NewOIDCAuthenticator(opts ...Option) *OIDCAuthenticator {
|
||||
options := newOptions(opts...)
|
||||
|
||||
return &OIDCAuthenticator{
|
||||
Logger: logger,
|
||||
tokenCache: &tokenCache,
|
||||
TokenCacheTTL: time.Duration(tokenCacheTTL),
|
||||
HTTPClient: oidcHTTPClient,
|
||||
OIDCIss: oidcIss,
|
||||
ProviderFunc: providerFunc,
|
||||
JWKSOptions: jwksOptions,
|
||||
AccessTokenVerifyMethod: accessTokenVerifyMethod,
|
||||
Logger: options.Logger,
|
||||
userInfoCache: options.Cache,
|
||||
DefaultTokenCacheTTL: options.DefaultAccessTokenTTL,
|
||||
HTTPClient: options.HTTPClient,
|
||||
OIDCIss: options.OIDCIss,
|
||||
ProviderFunc: options.OIDCProviderFunc,
|
||||
JWKSOptions: options.JWKS,
|
||||
AccessTokenVerifyMethod: options.AccessTokenVerifyMethod,
|
||||
providerLock: &sync.Mutex{},
|
||||
jwksLock: &sync.Mutex{},
|
||||
}
|
||||
@@ -51,8 +55,8 @@ type OIDCAuthenticator struct {
|
||||
Logger log.Logger
|
||||
HTTPClient *http.Client
|
||||
OIDCIss string
|
||||
tokenCache *osync.Cache
|
||||
TokenCacheTTL time.Duration
|
||||
userInfoCache store.Store
|
||||
DefaultTokenCacheTTL time.Duration
|
||||
ProviderFunc func() (OIDCProvider, error)
|
||||
AccessTokenVerifyMethod string
|
||||
JWKSOptions config.JWKS
|
||||
@@ -66,40 +70,60 @@ type OIDCAuthenticator struct {
|
||||
|
||||
func (m *OIDCAuthenticator) getClaims(token string, req *http.Request) (map[string]interface{}, error) {
|
||||
var claims map[string]interface{}
|
||||
hit := m.tokenCache.Load(token)
|
||||
if hit == nil {
|
||||
aClaims, err := m.verifyAccessToken(token)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to verify access token")
|
||||
}
|
||||
|
||||
oauth2Token := &oauth2.Token{
|
||||
AccessToken: token,
|
||||
}
|
||||
// usea 64 bytes long hash to have 256-bit collision resistance.
|
||||
hash := make([]byte, 64)
|
||||
sha3.ShakeSum256(hash, []byte(token))
|
||||
encodedHash := base64.URLEncoding.EncodeToString(hash)
|
||||
|
||||
userInfo, err := m.getProvider().UserInfo(
|
||||
context.WithValue(req.Context(), oauth2.HTTPClient, m.HTTPClient),
|
||||
oauth2.StaticTokenSource(oauth2Token),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get userinfo")
|
||||
record, err := m.userInfoCache.Read(encodedHash)
|
||||
if err != nil && err != store.ErrNotFound {
|
||||
m.Logger.Error().Err(err).Msg("could not read from userinfo cache")
|
||||
}
|
||||
if len(record) > 1 {
|
||||
if err = msgpack.Unmarshal(record[0].Value, &claims); err == nil {
|
||||
m.Logger.Debug().Interface("claims", claims).Msg("cache hit for userinfo")
|
||||
return claims, nil
|
||||
}
|
||||
if err := userInfo.Claims(&claims); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to unmarshal userinfo claims")
|
||||
}
|
||||
|
||||
expiration := m.extractExpiration(aClaims)
|
||||
m.tokenCache.Store(token, claims, expiration)
|
||||
|
||||
m.Logger.Debug().Interface("claims", claims).Interface("userInfo", userInfo).Time("expiration", expiration.UTC()).Msg("unmarshalled and cached userinfo")
|
||||
return claims, nil
|
||||
m.Logger.Error().Err(err).Msg("could not unmarshal userinfo")
|
||||
}
|
||||
aClaims, err := m.verifyAccessToken(token)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to verify access token")
|
||||
}
|
||||
|
||||
var ok bool
|
||||
if claims, ok = hit.V.(map[string]interface{}); !ok {
|
||||
return nil, errors.New("failed to cast claims from the cache")
|
||||
oauth2Token := &oauth2.Token{
|
||||
AccessToken: token,
|
||||
}
|
||||
m.Logger.Debug().Interface("claims", claims).Msg("cache hit for userinfo")
|
||||
|
||||
userInfo, err := m.getProvider().UserInfo(
|
||||
context.WithValue(req.Context(), oauth2.HTTPClient, m.HTTPClient),
|
||||
oauth2.StaticTokenSource(oauth2Token),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get userinfo")
|
||||
}
|
||||
if err := userInfo.Claims(&claims); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to unmarshal userinfo claims")
|
||||
}
|
||||
|
||||
expiration := m.extractExpiration(aClaims)
|
||||
go func() {
|
||||
if d, err := msgpack.Marshal(claims); err != nil {
|
||||
m.Logger.Error().Err(err).Msg("failed to marshal claims for userinfo cache")
|
||||
} else {
|
||||
err = m.userInfoCache.Write(&store.Record{
|
||||
Key: encodedHash,
|
||||
Value: d,
|
||||
Expiry: time.Until(expiration),
|
||||
})
|
||||
if err != nil {
|
||||
m.Logger.Error().Err(err).Msg("failed to write to userinfo cache")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
m.Logger.Debug().Interface("claims", claims).Msg("extracted claims")
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
@@ -145,7 +169,7 @@ func (m OIDCAuthenticator) verifyAccessTokenJWT(token string) (jwt.RegisteredCla
|
||||
// If the access token does not have an exp claim it will fallback to the configured
|
||||
// default expiration
|
||||
func (m OIDCAuthenticator) extractExpiration(aClaims jwt.RegisteredClaims) time.Time {
|
||||
defaultExpiration := time.Now().Add(m.TokenCacheTTL)
|
||||
defaultExpiration := time.Now().Add(m.DefaultTokenCacheTTL)
|
||||
if aClaims.ExpiresAt != nil {
|
||||
m.Logger.Debug().Str("exp", aClaims.ExpiresAt.String()).Msg("Expiration Time from access_token")
|
||||
return aClaims.ExpiresAt.Time
|
||||
|
||||
@@ -4,15 +4,14 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/proxy/pkg/user/backend"
|
||||
"github.com/owncloud/ocis/v2/services/proxy/pkg/userroles"
|
||||
|
||||
settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
|
||||
storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
|
||||
storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0"
|
||||
"github.com/owncloud/ocis/v2/services/proxy/pkg/config"
|
||||
"github.com/owncloud/ocis/v2/services/proxy/pkg/user/backend"
|
||||
"github.com/owncloud/ocis/v2/services/proxy/pkg/userroles"
|
||||
store "go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
@@ -52,10 +51,10 @@ type Options struct {
|
||||
AutoprovisionAccounts bool
|
||||
// EnableBasicAuth to allow basic auth
|
||||
EnableBasicAuth bool
|
||||
// UserinfoCacheSize defines the max number of entries in the userinfo cache, intended for the oidc_auth middleware
|
||||
UserinfoCacheSize int
|
||||
// UserinfoCacheTTL sets the max cache duration for the userinfo cache, intended for the oidc_auth middleware
|
||||
UserinfoCacheTTL time.Duration
|
||||
// DefaultAccessTokenTTL is used to calculate the expiration when an access token has no expiration set
|
||||
DefaultAccessTokenTTL time.Duration
|
||||
// Cache sets the access token cache store
|
||||
Cache store.Store
|
||||
// CredentialsByUserAgent sets the auth challenges on a per user-agent basis
|
||||
CredentialsByUserAgent map[string]string
|
||||
// AccessTokenVerifyMethod configures how access_tokens should be verified but the oidc_auth middleware.
|
||||
@@ -184,17 +183,17 @@ func EnableBasicAuth(enableBasicAuth bool) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// TokenCacheSize provides a function to set the TokenCacheSize
|
||||
func TokenCacheSize(size int) Option {
|
||||
// DefaultAccessTokenTTL provides a function to set the DefaultAccessTokenTTL
|
||||
func DefaultAccessTokenTTL(ttl time.Duration) Option {
|
||||
return func(o *Options) {
|
||||
o.UserinfoCacheSize = size
|
||||
o.DefaultAccessTokenTTL = ttl
|
||||
}
|
||||
}
|
||||
|
||||
// TokenCacheTTL provides a function to set the TokenCacheTTL
|
||||
func TokenCacheTTL(ttl time.Duration) Option {
|
||||
// Cache provides a function to set the Cache
|
||||
func Cache(val store.Store) Option {
|
||||
return func(o *Options) {
|
||||
o.UserinfoCacheTTL = ttl
|
||||
o.Cache = val
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -84,7 +84,7 @@ type OCISDriver struct {
|
||||
|
||||
// Cache holds cache config
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_SYSTEM_CACHE_STORE" desc:"Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', 'etcd' and 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_SYSTEM_CACHE_NODES" desc:"A comma separated list of addresses to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"STORAGE_SYSTEM_CACHE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when a 'memory', 'redis' or 'redis-sentinel' store is configured."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_SYSTEM_CACHE_STORE;OCIS_CACHE_STORE_TYPE" desc:"Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', 'etcd' and 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_SYSTEM_CACHE_NODES;OCIS_CACHE_STORE_ADDRESS" desc:"A comma separated list of nodes to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"STORAGE_SYSTEM_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
}
|
||||
|
||||
@@ -169,9 +169,9 @@ type Events struct {
|
||||
|
||||
// Cache holds cache config
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE" desc:"Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', and 'etcd'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_NODES" desc:"A comma separated list of addresses to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"STORAGE_USERS_CACHE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when a 'memory', 'redis' or 'redis-sentinel' store is configured."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_CACHE_STORE;STORAGE_USERS_CACHE_STORE_TYPE" desc:"Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', and 'etcd'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_CACHE_STORE_NODES;OCIS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"STORAGE_USERS_CACHE_DATABASE" desc:"The database name the configured store should use."`
|
||||
}
|
||||
|
||||
// S3Driver is the storage driver configuration when using 's3' storage driver
|
||||
|
||||
@@ -3,14 +3,12 @@ package command
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/events"
|
||||
"github.com/cs3org/reva/v2/pkg/events/stream"
|
||||
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/oklog/run"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
|
||||
ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/store"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/version"
|
||||
@@ -21,6 +19,7 @@ import (
|
||||
"github.com/owncloud/ocis/v2/services/userlog/pkg/metrics"
|
||||
"github.com/owncloud/ocis/v2/services/userlog/pkg/server/http"
|
||||
"github.com/urfave/cli/v2"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
// all events we care about
|
||||
@@ -74,10 +73,12 @@ func Server(cfg *config.Config) *cli.Command {
|
||||
}
|
||||
|
||||
st := store.Create(
|
||||
store.Type(cfg.Store.Type),
|
||||
store.Addresses(strings.Split(cfg.Store.Addresses, ",")...),
|
||||
store.Database(cfg.Store.Database),
|
||||
store.Table(cfg.Store.Table),
|
||||
store.Store(cfg.Persistence.Store),
|
||||
store.TTL(cfg.Persistence.TTL),
|
||||
store.Size(cfg.Persistence.Size),
|
||||
microstore.Nodes(cfg.Persistence.Nodes...),
|
||||
microstore.Database(cfg.Persistence.Database),
|
||||
microstore.Table(cfg.Persistence.Table),
|
||||
)
|
||||
|
||||
tm, err := pool.StringToTLSMode(cfg.GRPCClientTLS.Mode)
|
||||
@@ -93,7 +94,7 @@ func Server(cfg *config.Config) *cli.Command {
|
||||
return fmt.Errorf("could not get reva client: %s", err)
|
||||
}
|
||||
|
||||
hClient := ehsvc.NewEventHistoryService("com.owncloud.api.eventhistory", grpc.DefaultClient())
|
||||
hClient := ehsvc.NewEventHistoryService("com.owncloud.api.eventhistory", ogrpc.DefaultClient())
|
||||
|
||||
{
|
||||
server, err := http.Server(
|
||||
|
||||
@@ -2,6 +2,7 @@ package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
|
||||
)
|
||||
@@ -20,22 +21,23 @@ type Config struct {
|
||||
|
||||
TokenManager *TokenManager `yaml:"token_manager"`
|
||||
|
||||
MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;USERLOG_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary to access resources from other services."`
|
||||
RevaGateway string `yaml:"reva_gateway" env:"REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata"`
|
||||
TranslationPath string `yaml:"translation_path" env:"USERLOG_TRANSLATION_PATH" desc:"(optional) Set this to a path with custom translations to overwrite the builtin translations. See the documentation for more details."`
|
||||
Events Events `yaml:"events"`
|
||||
Store Store `yaml:"store"`
|
||||
MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;USERLOG_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary to access resources from other services."`
|
||||
RevaGateway string `yaml:"reva_gateway" env:"REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata"`
|
||||
TranslationPath string `yaml:"translation_path" env:"USERLOG_TRANSLATION_PATH" desc:"(optional) Set this to a path with custom translations to overwrite the builtin translations. See the documentation for more details."`
|
||||
Events Events `yaml:"events"`
|
||||
Persistence Persistence `yaml:"persistence"`
|
||||
|
||||
Context context.Context `yaml:"-"`
|
||||
}
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Type string `yaml:"type" env:"USERLOG_STORE_TYPE" desc:"The type of the userlog store. Supported values are: 'mem', 'ocmem', 'etcd', 'redis', 'nats-js', 'noop'. See the text description for details."`
|
||||
Addresses string `yaml:"addresses" env:"USERLOG_STORE_ADDRESSES" desc:"A comma separated list of addresses to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"USERLOG_STORE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when 'in-memory' stores or 'redis' is configured."`
|
||||
Table string `yaml:"table" env:"USERLOG_STORE_TABLE" desc:"(optional) The database table the store should use. This has no effect when 'in-memory' stores are configured."`
|
||||
Size int `yaml:"size" env:"USERLOG_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured. Defaults to 512."`
|
||||
// Persistence configures the store to use
|
||||
type Persistence struct {
|
||||
Store string `yaml:"store" env:"OCIS_PERSISTENT_STORE;USERLOG_STORE;USERLOG_STORE_TYPE" desc:"The type of the userlog store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_PERSISTENT_STORE_NODES;USERLOG_STORE_ADDRESSES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"USERLOG_STORE_DATABASE" desc:"The database name the configured store should use."`
|
||||
Table string `yaml:"table" env:"USERLOG_STORE_TABLE" desc:"The database table the store should use."`
|
||||
TTL time.Duration `yaml:"ttl" env:"OCIS_PERSISTENT_STORE_TTL;USERLOG_STORE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '336h' (2 weeks)."`
|
||||
Size int `yaml:"size" env:"OCIS_PERSISTENT_STORE_SIZE;USERLOG_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured. Defaults to 512."`
|
||||
}
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
|
||||
@@ -2,6 +2,7 @@ package defaults
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/structs"
|
||||
@@ -27,8 +28,11 @@ func DefaultConfig() *config.Config {
|
||||
Cluster: "ocis-cluster",
|
||||
EnableTLS: false,
|
||||
},
|
||||
Store: config.Store{
|
||||
Type: "mem",
|
||||
Persistence: config.Persistence{
|
||||
Store: "memory",
|
||||
Database: "userlog",
|
||||
Table: "events",
|
||||
TTL: time.Hour * 336,
|
||||
},
|
||||
RevaGateway: shared.DefaultRevaConfig().Address,
|
||||
HTTP: config.HTTP{
|
||||
|
||||
Reference in New Issue
Block a user