diff --git a/changelog/unreleased/change-presigned-key-store.md b/changelog/unreleased/change-presigned-key-store.md new file mode 100644 index 0000000000..06ad0bd6b0 --- /dev/null +++ b/changelog/unreleased/change-presigned-key-store.md @@ -0,0 +1,5 @@ +Change: change the default store for presigned keys to nats-js-kv + +We wrapped the store service in a micro store implementation and changed the default to the built-in NATS instance. + +https://github.com/owncloud/ocis/pull/8419 diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index 7830bcde6e..83d327f78e 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -57,7 +57,6 @@ import ( storageshares "github.com/owncloud/ocis/v2/services/storage-shares/pkg/command" storageSystem "github.com/owncloud/ocis/v2/services/storage-system/pkg/command" storageusers "github.com/owncloud/ocis/v2/services/storage-users/pkg/command" - store "github.com/owncloud/ocis/v2/services/store/pkg/command" thumbnails "github.com/owncloud/ocis/v2/services/thumbnails/pkg/command" userlog "github.com/owncloud/ocis/v2/services/userlog/pkg/command" users "github.com/owncloud/ocis/v2/services/users/pkg/command" @@ -245,11 +244,6 @@ func NewService(options ...Option) (*Service, error) { cfg.StorageUsers.Commons = cfg.Commons return storageusers.Execute(cfg.StorageUsers) }) - reg(3, opts.Config.Store.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error { - cfg.Store.Context = ctx - cfg.Store.Commons = cfg.Commons - return store.Execute(cfg.Store) - }) reg(3, opts.Config.Thumbnails.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error { cfg.Thumbnails.Context = ctx cfg.Thumbnails.Commons = cfg.Commons diff --git a/services/ocs/README.md b/services/ocs/README.md index 6425f82a6c..dd7a83f6d6 100644 --- a/services/ocs/README.md +++ b/services/ocs/README.md @@ -5,3 +5,20 @@ The `ocs` service (open collaboration services) serves one purpose: it has an en ## Signing-Keys Endpoint The `ocs` service contains an endpoint `/cloud/user/signing-key` on which a user can GET a signing key. Note, this functionality might be deprecated or moved in the future. + +## Signing-Keys Store + +To authenticate presigned URLs the proxy service needs to read the signing keys from a store that is populated by the ocs service. +Possible stores that can be configured via `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE` are: + - `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store) + - `redis-sentinel`: Stores data in a configured Redis Sentinel cluster. + - `ocisstoreservice`: Stores data in the legacy ocis store service. Requires setting `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` to `com.owncloud.api.store`. + +The `memory` or `ocmem` stores cannot be used as they do not share the memory from the ocs service signing key memory store, even in a single process. + +Make sure to configure the same store in the proxy service. + +Store specific notes: + - When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` in the form of `:/` like `10.10.0.200:26379/mymaster`. + - When using `nats-js-kv` it is recommended to set `PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` to the same value as `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES`. That way the proxy uses the same nats instance as the ocs service. + - When using `ocisstoreservice` the `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` must be set to the service name `com.owncloud.api.store`. It does not support TTL and stores the presigning keys indefinitely. Also, the store service needs to be started. diff --git a/services/ocs/pkg/config/config.go b/services/ocs/pkg/config/config.go index c8edf4c500..e6e3768a20 100644 --- a/services/ocs/pkg/config/config.go +++ b/services/ocs/pkg/config/config.go @@ -2,6 +2,7 @@ package config import ( "context" + "time" "github.com/owncloud/ocis/v2/ocis-pkg/shared" "go-micro.dev/v4/client" @@ -22,7 +23,18 @@ type Config struct { GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"` GrpcClient client.Client `yaml:"-"` + SigningKeys *SigningKeys `yaml:"signing_keys"` + TokenManager *TokenManager `yaml:"token_manager"` Context context.Context `yaml:"-"` } + +// SigningKeys is a store configuration. +type SigningKeys struct { + Store string `yaml:"store" env:"OCIS_CACHE_STORE;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE" desc:"The type of the signing key store. Supported values are: 'redis-sentinel' and 'nats-js-kv'. See the text description for details."` + Nodes []string `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES" desc:"A list of nodes to access the configured store. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` + TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_TTL" desc:"Default time to live for signing keys. See the Environment Variable Types description for more details."` + AuthUsername string `yaml:"username" env:"OCIS_CACHE_AUTH_USERNAME;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured."` + AuthPassword string `yaml:"password" env:"OCIS_CACHE_AUTH_PASSWORD;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured."` +} diff --git a/services/ocs/pkg/config/defaults/defaultconfig.go b/services/ocs/pkg/config/defaults/defaultconfig.go index 02a456b0d5..6a9ebec588 100644 --- a/services/ocs/pkg/config/defaults/defaultconfig.go +++ b/services/ocs/pkg/config/defaults/defaultconfig.go @@ -2,6 +2,7 @@ package defaults import ( "strings" + "time" "github.com/owncloud/ocis/v2/ocis-pkg/structs" "github.com/owncloud/ocis/v2/services/ocs/pkg/config" @@ -38,6 +39,11 @@ func DefaultConfig() *config.Config { Service: config.Service{ Name: "ocs", }, + SigningKeys: &config.SigningKeys{ + Store: "nats-js-kv", // signing keys are read by proxy, so we cannot use memory. It is not shared. + Nodes: []string{"127.0.0.1:9233"}, + TTL: time.Hour * 12, + }, } } diff --git a/services/ocs/pkg/server/http/server.go b/services/ocs/pkg/server/http/server.go index 58d9525a04..d7d8837fff 100644 --- a/services/ocs/pkg/server/http/server.go +++ b/services/ocs/pkg/server/http/server.go @@ -3,6 +3,7 @@ package http import ( "fmt" + "github.com/cs3org/reva/v2/pkg/store" chimiddleware "github.com/go-chi/chi/v5/middleware" "github.com/owncloud/ocis/v2/ocis-pkg/cors" "github.com/owncloud/ocis/v2/ocis-pkg/middleware" @@ -10,7 +11,9 @@ import ( "github.com/owncloud/ocis/v2/ocis-pkg/version" ocsmw "github.com/owncloud/ocis/v2/services/ocs/pkg/middleware" svc "github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0" + ocisstore "github.com/owncloud/ocis/v2/services/store/pkg/store" "go-micro.dev/v4" + microstore "go-micro.dev/v4/store" ) // Server initializes the http service and server. @@ -34,6 +37,24 @@ func Server(opts ...Option) (http.Service, error) { return http.Service{}, fmt.Errorf("could not initialize http service: %w", err) } + var signingKeyStore microstore.Store + if options.Config.SigningKeys.Store == "ocisstoreservice" { + signingKeyStore = ocisstore.NewStore( + microstore.Nodes(options.Config.SigningKeys.Nodes...), + microstore.Database("proxy"), + microstore.Table("signing-keys"), + ) + } else { + signingKeyStore = store.Create( + store.Store(options.Config.SigningKeys.Store), + store.TTL(options.Config.SigningKeys.TTL), + microstore.Nodes(options.Config.SigningKeys.Nodes...), + microstore.Database("proxy"), + microstore.Table("signing-keys"), + store.Authentication(options.Config.SigningKeys.AuthUsername, options.Config.SigningKeys.AuthPassword), + ) + } + handle := svc.NewService( svc.Logger(options.Logger), svc.Config(options.Config), @@ -56,6 +77,7 @@ func Server(opts ...Option) (http.Service, error) { middleware.Logger(options.Logger), ocsmw.LogTrace, ), + svc.Store(signingKeyStore), ) { diff --git a/services/ocs/pkg/service/v0/option.go b/services/ocs/pkg/service/v0/option.go index 40ccc9e1c7..9bf3c37ca9 100644 --- a/services/ocs/pkg/service/v0/option.go +++ b/services/ocs/pkg/service/v0/option.go @@ -5,6 +5,7 @@ import ( "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/ocs/pkg/config" + "go-micro.dev/v4/store" ) // Option defines a single option function. @@ -15,6 +16,7 @@ type Options struct { Logger log.Logger Config *config.Config Middleware []func(http.Handler) http.Handler + Store store.Store } // newOptions initializes the available default options. @@ -48,3 +50,10 @@ func Middleware(val ...func(http.Handler) http.Handler) Option { o.Middleware = val } } + +// Store provides a function to set the store option. +func Store(s store.Store) Option { + return func(o *Options) { + o.Store = s + } +} diff --git a/services/ocs/pkg/service/v0/service.go b/services/ocs/pkg/service/v0/service.go index 5e333415a9..207a33e0f5 100644 --- a/services/ocs/pkg/service/v0/service.go +++ b/services/ocs/pkg/service/v0/service.go @@ -14,6 +14,7 @@ import ( ocsm "github.com/owncloud/ocis/v2/services/ocs/pkg/middleware" "github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0/data" "github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0/response" + microstore "go-micro.dev/v4/store" ) // Service defines the service handlers. @@ -32,6 +33,7 @@ func NewService(opts ...Option) Service { config: options.Config, mux: m, logger: options.Logger, + store: options.Store, } m.Route(options.Config.HTTP.Root, func(r chi.Router) { @@ -61,6 +63,7 @@ type Ocs struct { config *config.Config logger log.Logger mux *chi.Mux + store microstore.Store } // ServeHTTP implements the Service interface. diff --git a/services/ocs/pkg/service/v0/users.go b/services/ocs/pkg/service/v0/users.go index 00bab3eb0a..190e9bccd4 100644 --- a/services/ocs/pkg/service/v0/users.go +++ b/services/ocs/pkg/service/v0/users.go @@ -3,15 +3,13 @@ package svc import ( "crypto/rand" "encoding/hex" + "errors" "net/http" - storemsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/store/v0" - storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0" - revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0/data" "github.com/owncloud/ocis/v2/services/ocs/pkg/service/v0/response" - merrors "go-micro.dev/v4/errors" + "go-micro.dev/v4/store" ) // GetSigningKey returns the signing key for the current user. It will create it on the fly if it does not exist @@ -28,24 +26,16 @@ func (o Ocs) GetSigningKey(w http.ResponseWriter, r *http.Request) { // use the user's UUID userID := u.Id.OpaqueId - c := storesvc.NewStoreService("com.owncloud.api.store", o.config.GrpcClient) - res, err := c.Read(r.Context(), &storesvc.ReadRequest{ - Options: &storemsg.ReadOptions{ - Database: "proxy", - Table: "signing-keys", - }, - Key: userID, - }) - if err == nil && len(res.Records) > 0 { + res, err := o.store.Read(userID) + if err == nil && len(res) > 0 { o.mustRender(w, r, response.DataRender(&data.SigningKey{ User: userID, - SigningKey: string(res.Records[0].Value), + SigningKey: string(res[0].Value), })) return } if err != nil { - e := merrors.Parse(err.Error()) - if e.Code == http.StatusNotFound { + if errors.Is(err, store.ErrNotFound) { // not found is ok, so we can continue and generate the key on the fly } else { o.logger.Error().Err(err).Msg("error reading from server") @@ -63,20 +53,8 @@ func (o Ocs) GetSigningKey(w http.ResponseWriter, r *http.Request) { } signingKey := hex.EncodeToString(key) - _, err = c.Write(r.Context(), &storesvc.WriteRequest{ - Options: &storemsg.WriteOptions{ - Database: "proxy", - Table: "signing-keys", - }, - Record: &storemsg.Record{ - Key: userID, - Value: []byte(signingKey), - // TODO Expiry? - }, - }) - + err = o.store.Write(&store.Record{Key: userID, Value: []byte(signingKey)}) if err != nil { - //o.logger.Error().Err(err).Msg("error writing key") o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, "could not persist signing key")) return } diff --git a/services/proxy/README.md b/services/proxy/README.md index d34424fbd9..a963287c18 100644 --- a/services/proxy/README.md +++ b/services/proxy/README.md @@ -153,6 +153,25 @@ Store specific notes: - When using `nats-js-kv` it is recommended to set `OCIS_CACHE_STORE_NODES` to the same value as `OCIS_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus. - When using the `nats-js-kv` store, it is possible to set `OCIS_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc. + +## Presigned Urls + +To authenticate presigned URLs the proxy service needs to read signing keys from a store that is populated by the ocs service. Possible stores are: + - `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store) + - `redis-sentinel`: Stores data in a configured Redis Sentinel cluster. + - `ocisstoreservice`: Stores data in the legacy ocis store service. Requires setting `PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` to `com.owncloud.api.store`. + +The `memory` or `ocmem` stores cannot be used as they do not share the memory from the ocs service signing key memory store, even in a single process. + +Make sure to configure the same store in the ocs service. + +Store specific notes: + - When using `redis-sentinel`, the Redis master to use is configured via e.g. `OCIS_CACHE_STORE_NODES` in the form of `:/` like `10.10.0.200:26379/mymaster`. + - When using `nats-js-kv` it is recommended to set `OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` to the same value as `PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES`. That way the ocs uses the same nats instance as the proxy service. + - When using the `nats-js-kv` store, it is possible to set `PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_DISABLE_PERSISTENCE` to instruct nats to not persist signing key data on disc. + - When using `ocisstoreservice` the `PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES` must be set to the service name `com.owncloud.api.store`. It does not support TTL and stores the presigning keys indefinitely. Also, the store service needs to be started. + + ## Special Settings When using the ocis IDP service instead of an external IDP: diff --git a/services/proxy/pkg/command/server.go b/services/proxy/pkg/command/server.go index 3363f048c1..a3739163e9 100644 --- a/services/proxy/pkg/command/server.go +++ b/services/proxy/pkg/command/server.go @@ -31,7 +31,6 @@ import ( "github.com/owncloud/ocis/v2/ocis-pkg/version" policiessvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/policies/v0" settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0" - storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0" "github.com/owncloud/ocis/v2/services/proxy/pkg/config" "github.com/owncloud/ocis/v2/services/proxy/pkg/config/parser" "github.com/owncloud/ocis/v2/services/proxy/pkg/logging" @@ -43,6 +42,7 @@ import ( proxyHTTP "github.com/owncloud/ocis/v2/services/proxy/pkg/server/http" "github.com/owncloud/ocis/v2/services/proxy/pkg/user/backend" "github.com/owncloud/ocis/v2/services/proxy/pkg/userroles" + ocisstore "github.com/owncloud/ocis/v2/services/store/pkg/store" ) // Server is the entrypoint for the server command. @@ -66,6 +66,24 @@ func Server(cfg *config.Config) *cli.Command { store.Authentication(cfg.OIDC.UserinfoCache.AuthUsername, cfg.OIDC.UserinfoCache.AuthPassword), ) + var signingKeyStore microstore.Store + if cfg.PreSignedURL.SigningKeys.Store == "ocisstoreservice" { + signingKeyStore = ocisstore.NewStore( + microstore.Nodes(cfg.PreSignedURL.SigningKeys.Nodes...), + microstore.Database("proxy"), + microstore.Table("signing-keys"), + ) + } else { + signingKeyStore = store.Create( + store.Store(cfg.PreSignedURL.SigningKeys.Store), + store.TTL(cfg.PreSignedURL.SigningKeys.TTL), + microstore.Nodes(cfg.PreSignedURL.SigningKeys.Nodes...), + microstore.Database("proxy"), + microstore.Table("signing-keys"), + store.Authentication(cfg.PreSignedURL.SigningKeys.AuthUsername, cfg.PreSignedURL.SigningKeys.AuthPassword), + ) + } + logger := logging.Configure(cfg.Service.Name, cfg.Log) traceProvider, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) if err != nil { @@ -130,7 +148,7 @@ func Server(cfg *config.Config) *cli.Command { } { - middlewares := loadMiddlewares(ctx, logger, cfg, userInfoCache, traceProvider, *m) + middlewares := loadMiddlewares(ctx, logger, cfg, userInfoCache, signingKeyStore, traceProvider, *m) server, err := proxyHTTP.Server( proxyHTTP.Handler(lh.handler()), proxyHTTP.Logger(logger), @@ -271,7 +289,7 @@ func (h *StaticRouteHandler) backchannelLogout(w http.ResponseWriter, r *http.Re render.JSON(w, r, nil) } -func loadMiddlewares(ctx context.Context, logger log.Logger, cfg *config.Config, userInfoCache microstore.Store, traceProvider trace.TracerProvider, metrics metrics.Metrics) alice.Chain { +func loadMiddlewares(ctx context.Context, logger log.Logger, cfg *config.Config, userInfoCache, signingKeyStore microstore.Store, traceProvider trace.TracerProvider, metrics metrics.Metrics) alice.Chain { rolesClient := settingssvc.NewRoleService("com.owncloud.api.settings", cfg.GrpcClient) policiesProviderClient := policiessvc.NewPoliciesProviderService("com.owncloud.api.policies", cfg.GrpcClient) gatewaySelector, err := pool.GatewaySelector( @@ -322,13 +340,6 @@ func loadMiddlewares(ctx context.Context, logger log.Logger, cfg *config.Config, logger.Fatal().Msgf("Invalid role assignment driver '%s'", cfg.RoleAssignment.Driver) } - storeClient := storesvc.NewStoreService("com.owncloud.api.store", cfg.GrpcClient) - if err != nil { - logger.Error().Err(err). - Str("gateway", cfg.Reva.Address). - Msg("Failed to create reva gateway service client") - } - oidcHTTPClient := &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ @@ -373,7 +384,7 @@ func loadMiddlewares(ctx context.Context, logger log.Logger, cfg *config.Config, PreSignedURLConfig: cfg.PreSignedURL, UserProvider: userProvider, UserRoleAssigner: roleAssigner, - Store: storeClient, + Store: signingKeyStore, Now: time.Now, }) diff --git a/services/proxy/pkg/config/config.go b/services/proxy/pkg/config/config.go index 8ddb8c6ff4..264fcefe8e 100644 --- a/services/proxy/pkg/config/config.go +++ b/services/proxy/pkg/config/config.go @@ -166,8 +166,19 @@ type StaticSelectorConf struct { // PreSignedURL is the config for the presigned url middleware type PreSignedURL struct { - AllowedHTTPMethods []string `yaml:"allowed_http_methods"` - Enabled bool `yaml:"enabled" env:"PROXY_ENABLE_PRESIGNEDURLS" desc:"Allow OCS to get a signing key to sign requests."` + AllowedHTTPMethods []string `yaml:"allowed_http_methods"` + Enabled bool `yaml:"enabled" env:"PROXY_ENABLE_PRESIGNEDURLS" desc:"Allow OCS to get a signing key to sign requests."` + SigningKeys *SigningKeys `yaml:"signing_keys"` +} + +// SigningKeys is a store configuration. +type SigningKeys struct { + Store string `yaml:"store" env:"OCIS_CACHE_STORE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE" desc:"The type of the signing key store. Supported values are: 'redis-sentinel', 'nats-js-kv' and 'ocisstoreservice' (deprecated). See the text description for details."` + Nodes []string `yaml:"addresses" env:"OCIS_CACHE_STORE_NODES;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES" desc:"A list of nodes to access the configured store. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details."` + TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_TTL" desc:"Default time to live for signing keys. See the Environment Variable Types description for more details."` + DisablePersistence bool `yaml:"disable_persistence" env:"OCIS_CACHE_DISABLE_PERSISTENCE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_DISABLE_PERSISTENCE" desc:"Disables persistence of the store. Only applies when store type 'nats-js-kv' is configured. Defaults to true."` + AuthUsername string `yaml:"username" env:"OCIS_CACHE_AUTH_USERNAME;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured."` + AuthPassword string `yaml:"password" env:"OCIS_CACHE_AUTH_PASSWORD;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured."` } // ClaimsSelectorConf is the config for the claims-selector diff --git a/services/proxy/pkg/config/defaults/defaultconfig.go b/services/proxy/pkg/config/defaults/defaultconfig.go index 9c07ecb884..b6a9fd262a 100644 --- a/services/proxy/pkg/config/defaults/defaultconfig.go +++ b/services/proxy/pkg/config/defaults/defaultconfig.go @@ -73,6 +73,12 @@ func DefaultConfig() *config.Config { PreSignedURL: config.PreSignedURL{ AllowedHTTPMethods: []string{"GET"}, Enabled: true, + SigningKeys: &config.SigningKeys{ + Store: "nats-js-kv", // signing keys are written by ocs, so we cannot use memory. It is not shared. + Nodes: []string{"127.0.0.1:9233"}, + TTL: time.Hour * 12, + DisablePersistence: true, + }, }, AccountBackend: "cs3", UserOIDCClaim: "preferred_username", diff --git a/services/proxy/pkg/middleware/options.go b/services/proxy/pkg/middleware/options.go index 68c567a673..917f5ca311 100644 --- a/services/proxy/pkg/middleware/options.go +++ b/services/proxy/pkg/middleware/options.go @@ -5,19 +5,16 @@ import ( "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - "go-micro.dev/v4/store" - "go.opentelemetry.io/otel/trace" - "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" - "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/ocis-pkg/oidc" policiessvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/policies/v0" settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0" - storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0" "github.com/owncloud/ocis/v2/services/proxy/pkg/config" "github.com/owncloud/ocis/v2/services/proxy/pkg/user/backend" "github.com/owncloud/ocis/v2/services/proxy/pkg/userroles" + "go-micro.dev/v4/store" + "go.opentelemetry.io/otel/trace" ) // Option defines a single option function. @@ -45,8 +42,6 @@ type Options struct { OIDCIss string // RevaGatewaySelector to send requests to the reva gateway RevaGatewaySelector pool.Selectable[gateway.GatewayAPIClient] - // Store for persisting data - Store storesvc.StoreService // PreSignedURLConfig to configure the middleware PreSignedURLConfig config.PreSignedURL // UserOIDCClaim to read from the oidc claims @@ -151,13 +146,6 @@ func WithRevaGatewaySelector(val pool.Selectable[gateway.GatewayAPIClient]) Opti } } -// Store provides a function to set the store option. -func Store(sc storesvc.StoreService) Option { - return func(o *Options) { - o.Store = sc - } -} - // PreSignedURLConfig provides a function to set the PreSignedURL config func PreSignedURLConfig(cfg config.PreSignedURL) Option { return func(o *Options) { diff --git a/services/proxy/pkg/middleware/signed_url_auth.go b/services/proxy/pkg/middleware/signed_url_auth.go index b3c842a26f..bc2b72aa2b 100644 --- a/services/proxy/pkg/middleware/signed_url_auth.go +++ b/services/proxy/pkg/middleware/signed_url_auth.go @@ -1,7 +1,6 @@ package middleware import ( - "context" "crypto/sha512" "encoding/hex" "errors" @@ -13,11 +12,10 @@ import ( revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/owncloud/ocis/v2/ocis-pkg/log" - storemsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/store/v0" - storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0" "github.com/owncloud/ocis/v2/services/proxy/pkg/config" "github.com/owncloud/ocis/v2/services/proxy/pkg/user/backend" "github.com/owncloud/ocis/v2/services/proxy/pkg/userroles" + microstore "go-micro.dev/v4/store" "golang.org/x/crypto/pbkdf2" ) @@ -46,7 +44,7 @@ type SignedURLAuthenticator struct { PreSignedURLConfig config.PreSignedURL UserProvider backend.UserBackend UserRoleAssigner userroles.UserRoleAssigner - Store storesvc.StoreService + Store microstore.Store Now func() time.Time } @@ -147,17 +145,17 @@ func (m SignedURLAuthenticator) urlIsExpired(query url.Values) (err error) { func (m SignedURLAuthenticator) signatureIsValid(req *http.Request) (err error) { c := revactx.ContextMustGetUser(req.Context()) - signingKey, err := m.getSigningKey(req.Context(), c.Id.OpaqueId) + signingKey, err := m.Store.Read(c.Id.OpaqueId) if err != nil { m.Logger.Error().Err(err).Msg("could not retrieve signing key") return err } - if len(signingKey) == 0 { + if len(signingKey[0].Value) == 0 { m.Logger.Error().Err(err).Msg("signing key empty") return err } u := m.buildUrlToSign(req) - computedSignature := m.createSignature(u, signingKey) + computedSignature := m.createSignature(u, signingKey[0].Value) signatureInURL := req.URL.Query().Get(_paramOCSignature) if computedSignature == signatureInURL { return nil @@ -206,21 +204,6 @@ func (m SignedURLAuthenticator) createSignature(url string, signingKey []byte) s return hex.EncodeToString(hash) } -func (m SignedURLAuthenticator) getSigningKey(ctx context.Context, ocisID string) ([]byte, error) { - res, err := m.Store.Read(ctx, &storesvc.ReadRequest{ - Options: &storemsg.ReadOptions{ - Database: "proxy", - Table: "signing-keys", - }, - Key: ocisID, - }) - if err != nil || len(res.Records) < 1 { - return nil, err - } - - return res.Records[0].Value, nil -} - // Authenticate implements the authenticator interface to authenticate requests via signed URL auth. func (m SignedURLAuthenticator) Authenticate(r *http.Request) (*http.Request, bool) { if !m.shouldServe(r) { diff --git a/services/proxy/pkg/middleware/signed_url_auth_test.go b/services/proxy/pkg/middleware/signed_url_auth_test.go index a14edc01f4..dbbea75d62 100644 --- a/services/proxy/pkg/middleware/signed_url_auth_test.go +++ b/services/proxy/pkg/middleware/signed_url_auth_test.go @@ -2,16 +2,15 @@ package middleware import ( "context" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - revactx "github.com/cs3org/reva/v2/pkg/ctx" - storemsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/store/v0" - v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0" - "github.com/owncloud/ocis/v2/services/proxy/pkg/config" - "github.com/stretchr/testify/assert" - "go-micro.dev/v4/client" "net/http/httptest" "testing" "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + revactx "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/owncloud/ocis/v2/services/proxy/pkg/config" + "github.com/stretchr/testify/assert" + "go-micro.dev/v4/store" ) func TestSignedURLAuth_shouldServe(t *testing.T) { @@ -159,36 +158,6 @@ func TestSignedURLAuth_createSignature(t *testing.T) { } } -type MyStoreService struct { -} - -func (s MyStoreService) Read(_ context.Context, _ *v0.ReadRequest, _ ...client.CallOption) (*v0.ReadResponse, error) { - r := v0.ReadResponse{ - Records: []*storemsg.Record{{ - Key: "foo", - Value: []byte("1234567890"), - Expiry: 0, - Metadata: nil, - }}, - } - return &r, nil -} -func (s MyStoreService) Write(_ context.Context, _ *v0.WriteRequest, _ ...client.CallOption) (*v0.WriteResponse, error) { - return nil, nil -} -func (s MyStoreService) Delete(_ context.Context, _ *v0.DeleteRequest, _ ...client.CallOption) (*v0.DeleteResponse, error) { - return nil, nil -} -func (s MyStoreService) List(_ context.Context, _ *v0.ListRequest, _ ...client.CallOption) (v0.Store_ListService, error) { - return nil, nil -} -func (s MyStoreService) Databases(_ context.Context, _ *v0.DatabasesRequest, _ ...client.CallOption) (*v0.DatabasesResponse, error) { - return nil, nil -} -func (s MyStoreService) Tables(_ context.Context, _ *v0.TablesRequest, _ ...client.CallOption) (*v0.TablesResponse, error) { - return nil, nil -} - func TestSignedURLAuth_validate(t *testing.T) { nowFunc := func() time.Time { t, _ := time.Parse(time.RFC3339, "2020-02-02T12:30:00.000Z") @@ -198,13 +167,18 @@ func TestSignedURLAuth_validate(t *testing.T) { AllowedHTTPMethods: []string{"get"}, Enabled: true, } - store := MyStoreService{} pua := SignedURLAuthenticator{ PreSignedURLConfig: cfg, - Store: store, + Store: store.NewMemoryStore(), Now: nowFunc, } + pua.Store.Write(&store.Record{ + Key: "useri", + Value: []byte("1234567890"), + Metadata: nil, + }) + tests := []struct { now string url string diff --git a/services/store/pkg/store/options.go b/services/store/pkg/store/options.go new file mode 100644 index 0000000000..949425c795 --- /dev/null +++ b/services/store/pkg/store/options.go @@ -0,0 +1,20 @@ +package store + +import ( + "context" + + "go-micro.dev/v4/client" + "go-micro.dev/v4/store" +) + +type grpcClientContextKey struct{} + +// WithGRPCClient sets the grpc client +func WithGRPCClient(c client.Client) store.Option { + return func(o *store.Options) { + if o.Context == nil { + o.Context = context.Background() + } + o.Context = context.WithValue(o.Context, grpcClientContextKey{}, c) + } +} diff --git a/services/store/pkg/store/store.go b/services/store/pkg/store/store.go new file mode 100644 index 0000000000..47e7ee57df --- /dev/null +++ b/services/store/pkg/store/store.go @@ -0,0 +1,189 @@ +package store + +import ( + "context" + "errors" + "net/http" + "time" + + "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc" + storemsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/store/v0" + storesvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/store/v0" + "go-micro.dev/v4/client" + merrors "go-micro.dev/v4/errors" + "go-micro.dev/v4/logger" + "go-micro.dev/v4/store" + "go-micro.dev/v4/util/cmd" +) + +// DefaultDatabase is the namespace that the store +// will use if no namespace is provided. +var ( + DefaultDatabase = "proxy" + DefaultTable = "signing-keys" +) + +type oss struct { + ctx context.Context + options store.Options + svc storesvc.StoreService +} + +func init() { + cmd.DefaultStores["ocisstoreservice"] = NewStore +} + +// NewStore returns a micro store.Store wrapper to access the micro store service. +// It only implements the minimal Read and Write options that are used by the proxy and ocs services +// Deprecated: use a different micro.Store implementation like nats-js-ks +func NewStore(opts ...store.Option) store.Store { + options := store.Options{ + Context: context.Background(), + Database: DefaultDatabase, + Table: DefaultTable, + Logger: logger.DefaultLogger, + Nodes: []string{"com.owncloud.api.store"}, + } + + for _, o := range opts { + o(&options) + } + + c, ok := options.Context.Value(grpcClientContextKey{}).(client.Client) + if !ok { + var err error + c, err = grpc.NewClient() + if err != nil { + options.Logger.Fields(map[string]interface{}{"err": err}).Log(logger.FatalLevel, "ocisstoreservice could not create new grpc client") + } + } + svc := storesvc.NewStoreService(options.Nodes[0], c) + + s := &oss{ + ctx: context.Background(), + options: options, + svc: svc, + } + + return s +} + +// Init initializes the store by configuring a storeservice and initializing +// a grpc client if it has not been passed as a context option. +func (s *oss) Init(opts ...store.Option) error { + for _, o := range opts { + o(&s.options) + } + return s.configure() +} + +func (s *oss) configure() error { + c, ok := s.options.Context.Value(grpcClientContextKey{}).(client.Client) + if !ok { + var err error + c, err = grpc.NewClient() + if err != nil { + logger.Fatal("ocisstoreservice could not create new grpc client:", err) + } + } + if len(s.options.Nodes) < 1 { + return errors.New("no node configured") + } + s.svc = storesvc.NewStoreService(s.options.Nodes[0], c) + return nil +} + +// Options allows you to view the current options. +func (s *oss) Options() store.Options { + return s.options +} + +// Read takes a single key name and optional ReadOptions. It returns matching []*Record or an error. +// Only database and table options are used. +func (s *oss) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { + options := store.ReadOptions{ + Database: s.options.Database, + Table: s.options.Table, + } + + for _, o := range opts { + o(&options) + } + + res, err := s.svc.Read(context.Background(), &storesvc.ReadRequest{ + Options: &storemsg.ReadOptions{ + Database: options.Database, + Table: options.Table, + // Other options ignored + }, + Key: key, + }) + + if err != nil { + e := merrors.Parse(err.Error()) + if e.Code == http.StatusNotFound { + return nil, store.ErrNotFound + } + return nil, err + } + + records := make([]*store.Record, 0, len(res.Records)) + for _, record := range res.Records { + r := &store.Record{ + Key: record.Key, + Value: record.Value, + Metadata: map[string]interface{}{}, + Expiry: time.Duration(record.Expiry), + } + for k, v := range record.Metadata { + r.Metadata[k] = v.Value // we only support string + } + records = append(records, r) + } + return records, nil +} + +// Write() writes a record to the store, and returns an error if the record was not written. +func (s *oss) Write(r *store.Record, opts ...store.WriteOption) error { + options := store.WriteOptions{ + Database: s.options.Database, + Table: s.options.Table, + } + + for _, o := range opts { + o(&options) + } + _, err := s.svc.Write(context.Background(), &storesvc.WriteRequest{ + Options: &storemsg.WriteOptions{ + Database: options.Database, + Table: options.Table, + }, + Record: &storemsg.Record{ + Key: r.Key, + Value: r.Value, + // No expiry supported + }, + }) + + return err +} + +// Delete is not implemented for the ocis service +func (s *oss) Delete(key string, opts ...store.DeleteOption) error { + return errors.ErrUnsupported +} + +// List is not implemented for the ocis service +func (s *oss) List(opts ...store.ListOption) ([]string, error) { + return nil, errors.ErrUnsupported +} + +// Close does nothing +func (s *oss) Close() error { + return nil +} + +// String returns the name of the implementation. +func (s *oss) String() string { + return "ocisstoreservice" +}