From 732228ed883449a72fc95163ba1a43547227dcc9 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 5 Sep 2023 10:25:18 +0200 Subject: [PATCH 1/6] add clientlog service Signed-off-by: jkoberg --- .drone.star | 1 + Makefile | 1 + changelog/unreleased/clientlog-service.md | 5 + .../general-info/new-service-checklist.md | 2 + ocis-pkg/config/config.go | 2 + ocis-pkg/config/defaultconfig.go | 2 + ocis/pkg/command/services.go | 6 + ocis/pkg/init/init.go | 8 + ocis/pkg/runtime/service/service.go | 6 + services/clientlog/Makefile | 38 +++++ services/clientlog/README.md | 14 ++ services/clientlog/cmd/clientlog/main.go | 14 ++ services/clientlog/pkg/command/health.go | 18 +++ services/clientlog/pkg/command/root.go | 34 +++++ services/clientlog/pkg/command/server.go | 137 ++++++++++++++++++ services/clientlog/pkg/command/version.go | 19 +++ services/clientlog/pkg/config/config.go | 49 +++++++ services/clientlog/pkg/config/debug.go | 9 ++ .../pkg/config/defaults/defaultconfig.go | 80 ++++++++++ services/clientlog/pkg/config/log.go | 9 ++ services/clientlog/pkg/config/parser/parse.go | 43 ++++++ services/clientlog/pkg/config/service.go | 6 + services/clientlog/pkg/config/tracing.go | 21 +++ services/clientlog/pkg/logging/logging.go | 17 +++ services/clientlog/pkg/metrics/metrics.go | 35 +++++ services/clientlog/pkg/service/options.go | 65 +++++++++ services/clientlog/pkg/service/service.go | 134 +++++++++++++++++ .../pkg/service/service_suit_test.go | 26 ++++ .../clientlog/pkg/service/service_test.go | 3 + services/sse/README.md | 7 + services/userlog/README.md | 7 + 31 files changed, 818 insertions(+) create mode 100644 changelog/unreleased/clientlog-service.md create mode 100644 services/clientlog/Makefile create mode 100644 services/clientlog/README.md create mode 100644 services/clientlog/cmd/clientlog/main.go create mode 100644 services/clientlog/pkg/command/health.go create mode 100644 services/clientlog/pkg/command/root.go create mode 100644 services/clientlog/pkg/command/server.go create mode 100644 services/clientlog/pkg/command/version.go create mode 100644 services/clientlog/pkg/config/config.go create mode 100644 services/clientlog/pkg/config/debug.go create mode 100644 services/clientlog/pkg/config/defaults/defaultconfig.go create mode 100644 services/clientlog/pkg/config/log.go create mode 100644 services/clientlog/pkg/config/parser/parse.go create mode 100644 services/clientlog/pkg/config/service.go create mode 100644 services/clientlog/pkg/config/tracing.go create mode 100644 services/clientlog/pkg/logging/logging.go create mode 100644 services/clientlog/pkg/metrics/metrics.go create mode 100644 services/clientlog/pkg/service/options.go create mode 100644 services/clientlog/pkg/service/service.go create mode 100644 services/clientlog/pkg/service/service_suit_test.go create mode 100644 services/clientlog/pkg/service/service_test.go diff --git a/.drone.star b/.drone.star index ba83888cb9..563272dd34 100644 --- a/.drone.star +++ b/.drone.star @@ -62,6 +62,7 @@ config = { "services/auth-bearer", "services/auth-machine", "services/auth-service", + "services/clientlog", "services/eventhistory", "services/frontend", "services/gateway", diff --git a/Makefile b/Makefile index 1b9cd18ca3..b83a0c3233 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ OCIS_MODULES = \ services/auth-bearer \ services/auth-machine \ services/auth-service \ + services/clientlog \ services/eventhistory \ services/frontend \ services/gateway \ diff --git a/changelog/unreleased/clientlog-service.md b/changelog/unreleased/clientlog-service.md new file mode 100644 index 0000000000..e8b2f111bd --- /dev/null +++ b/changelog/unreleased/clientlog-service.md @@ -0,0 +1,5 @@ +Enhancement: Introduce clientlog service + +Add the clientlog service which will send machine readable notifications to clients + +https://github.com/owncloud/ocis/pull/7217 diff --git a/docs/services/general-info/new-service-checklist.md b/docs/services/general-info/new-service-checklist.md index 45dbf364bd..dd50d3f1e7 100644 --- a/docs/services/general-info/new-service-checklist.md +++ b/docs/services/general-info/new-service-checklist.md @@ -28,6 +28,8 @@ Use this checklist with copy/paste in your PR - right from the beginning. It ren - [ ] Make the service startable for binary and individual startup: - For single binary add service to `ocis/pkg/runtime` - For individual startup add service to `ocis/pkg/commands` + - Add the service config to `ocis-pkg/config/defaultconfig.go` +- [ ] If the service is using service accounts, add it to `ocis/pkg/init/init.go` - [ ] Add the service to `.drone.star` to enable CI. - [ ] Inform doc team in an _early stage_ to review the readme AND the environment variables created. - The description must reflect the behaviour AND usually has a positive code quality impact. diff --git a/ocis-pkg/config/config.go b/ocis-pkg/config/config.go index 77fec4f6fa..8473d3877b 100644 --- a/ocis-pkg/config/config.go +++ b/ocis-pkg/config/config.go @@ -10,6 +10,7 @@ import ( authbearer "github.com/owncloud/ocis/v2/services/auth-bearer/pkg/config" authmachine "github.com/owncloud/ocis/v2/services/auth-machine/pkg/config" authservice "github.com/owncloud/ocis/v2/services/auth-service/pkg/config" + clientlog "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" eventhistory "github.com/owncloud/ocis/v2/services/eventhistory/pkg/config" frontend "github.com/owncloud/ocis/v2/services/frontend/pkg/config" gateway "github.com/owncloud/ocis/v2/services/gateway/pkg/config" @@ -85,6 +86,7 @@ type Config struct { AuthBearer *authbearer.Config `yaml:"auth_bearer"` AuthMachine *authmachine.Config `yaml:"auth_machine"` AuthService *authservice.Config `yaml:"auth_service"` + Clientlog *clientlog.Config `yaml:"clientlog"` EventHistory *eventhistory.Config `yaml:"eventhistory"` Frontend *frontend.Config `yaml:"frontend"` Gateway *gateway.Config `yaml:"gateway"` diff --git a/ocis-pkg/config/defaultconfig.go b/ocis-pkg/config/defaultconfig.go index 8639821bc7..3d068b09b9 100644 --- a/ocis-pkg/config/defaultconfig.go +++ b/ocis-pkg/config/defaultconfig.go @@ -9,6 +9,7 @@ import ( authbearer "github.com/owncloud/ocis/v2/services/auth-bearer/pkg/config/defaults" authmachine "github.com/owncloud/ocis/v2/services/auth-machine/pkg/config/defaults" authservice "github.com/owncloud/ocis/v2/services/auth-service/pkg/config/defaults" + clientlog "github.com/owncloud/ocis/v2/services/clientlog/pkg/config/defaults" eventhistory "github.com/owncloud/ocis/v2/services/eventhistory/pkg/config/defaults" frontend "github.com/owncloud/ocis/v2/services/frontend/pkg/config/defaults" gateway "github.com/owncloud/ocis/v2/services/gateway/pkg/config/defaults" @@ -57,6 +58,7 @@ func DefaultConfig() *Config { AuthBearer: authbearer.DefaultConfig(), AuthMachine: authmachine.DefaultConfig(), AuthService: authservice.DefaultConfig(), + Clientlog: clientlog.DefaultConfig(), EventHistory: eventhistory.DefaultConfig(), Frontend: frontend.DefaultConfig(), Gateway: gateway.DefaultConfig(), diff --git a/ocis/pkg/command/services.go b/ocis/pkg/command/services.go index da634e2dfa..b745f5d258 100644 --- a/ocis/pkg/command/services.go +++ b/ocis/pkg/command/services.go @@ -16,6 +16,7 @@ import ( authbearer "github.com/owncloud/ocis/v2/services/auth-bearer/pkg/command" authmachine "github.com/owncloud/ocis/v2/services/auth-machine/pkg/command" authservice "github.com/owncloud/ocis/v2/services/auth-service/pkg/command" + clientlog "github.com/owncloud/ocis/v2/services/clientlog/pkg/command" eventhistory "github.com/owncloud/ocis/v2/services/eventhistory/pkg/command" frontend "github.com/owncloud/ocis/v2/services/frontend/pkg/command" gateway "github.com/owncloud/ocis/v2/services/gateway/pkg/command" @@ -89,6 +90,11 @@ var svccmds = []register.Command{ cfg.AuthService.Commons = cfg.Commons }) }, + func(cfg *config.Config) *cli.Command { + return ServiceCommand(cfg, cfg.Clientlog.Service.Name, clientlog.GetCommands(cfg.Clientlog), func(c *config.Config) { + cfg.Clientlog.Commons = cfg.Commons + }) + }, func(cfg *config.Config) *cli.Command { return ServiceCommand(cfg, cfg.EventHistory.Service.Name, eventhistory.GetCommands(cfg.EventHistory), func(c *config.Config) { cfg.EventHistory.Commons = cfg.Commons diff --git a/ocis/pkg/init/init.go b/ocis/pkg/init/init.go index b49352e915..6a1fa4809d 100644 --- a/ocis/pkg/init/init.go +++ b/ocis/pkg/init/init.go @@ -142,6 +142,10 @@ type AuthService struct { ServiceAccount ServiceAccount `yaml:"service_account"` } +type Clientlog struct { + ServiceAccount ServiceAccount `yaml:"service_account"` +} + type Nats struct { // The nats config has a field called nats Nats struct { @@ -194,6 +198,7 @@ type OcisConfig struct { Gateway Gateway Userlog Userlog AuthService AuthService `yaml:"auth_service"` + Clientlog Clientlog } func checkConfigPath(configPath string) error { @@ -381,6 +386,9 @@ func CreateConfig(insecure, forceOverwrite bool, configPath, adminPassword strin Frontend: FrontendService{ ServiceAccount: serviceAccount, }, + Clientlog: Clientlog{ + ServiceAccount: serviceAccount, + }, } if insecure { diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index 1f43db921e..9687867fd0 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -27,6 +27,7 @@ import ( authbasic "github.com/owncloud/ocis/v2/services/auth-basic/pkg/command" authmachine "github.com/owncloud/ocis/v2/services/auth-machine/pkg/command" authservice "github.com/owncloud/ocis/v2/services/auth-service/pkg/command" + clientlog "github.com/owncloud/ocis/v2/services/clientlog/pkg/command" eventhistory "github.com/owncloud/ocis/v2/services/eventhistory/pkg/command" frontend "github.com/owncloud/ocis/v2/services/frontend/pkg/command" gateway "github.com/owncloud/ocis/v2/services/gateway/pkg/command" @@ -141,6 +142,11 @@ func NewService(options ...Option) (*Service, error) { cfg.AuthService.Commons = cfg.Commons return authservice.Execute(cfg.AuthService) }) + reg(opts.Config.Clientlog.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error { + cfg.Clientlog.Context = ctx + cfg.Clientlog.Commons = cfg.Commons + return clientlog.Execute(cfg.Clientlog) + }) reg(opts.Config.EventHistory.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error { cfg.EventHistory.Context = ctx cfg.EventHistory.Commons = cfg.Commons diff --git a/services/clientlog/Makefile b/services/clientlog/Makefile new file mode 100644 index 0000000000..5cf6d0b10f --- /dev/null +++ b/services/clientlog/Makefile @@ -0,0 +1,38 @@ +SHELL := bash +NAME := clientlog + +include ../../.make/recursion.mk + +############ tooling ############ +ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI +include ../../.bingo/Variables.mk +endif + +############ go tooling ############ +include ../../.make/go.mk + +############ release ############ +include ../../.make/release.mk + +############ docs generate ############ +include ../../.make/docs.mk + +.PHONY: docs-generate +docs-generate: config-docs-generate + +############ generate ############ +include ../../.make/generate.mk + +.PHONY: ci-go-generate +ci-go-generate: $(MOCKERY) # CI runs ci-node-generate automatically before this target + $(MOCKERY) --dir ../../protogen/gen/ocis/services/eventhistory/v0 --case underscore --name EventHistoryService + +.PHONY: ci-node-generate +ci-node-generate: + +############ licenses ############ +.PHONY: ci-node-check-licenses +ci-node-check-licenses: + +.PHONY: ci-node-save-licenses +ci-node-save-licenses: diff --git a/services/clientlog/README.md b/services/clientlog/README.md new file mode 100644 index 0000000000..9421f5b3bd --- /dev/null +++ b/services/clientlog/README.md @@ -0,0 +1,14 @@ +# Clientlog service + +The `clientlog` service is responsible for composing machine readable notifications for clients + +## The `...log` service ecosystem + +`...log` services (`userlog`, `clientlog`) are responsible for composing notifications for a certain audience. + - `userlog` service translates and adjust messages to be human readable + - `clientlog` service composes machine readable messages so clients can act without needing to query the server + - `sse` service is only responsible for sending these messages. It does not care about their form or language + +## `clientlog` events + +The messages the `clientlog` service sends are meant to be used by clients, not by users. The client might for example be informed that a file is finished postprocessing, so it can make the file available to the user without needing to make another call to the server. diff --git a/services/clientlog/cmd/clientlog/main.go b/services/clientlog/cmd/clientlog/main.go new file mode 100644 index 0000000000..ce0d4a4aff --- /dev/null +++ b/services/clientlog/cmd/clientlog/main.go @@ -0,0 +1,14 @@ +package main + +import ( + "os" + + "github.com/owncloud/ocis/v2/services/clientlog/pkg/command" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config/defaults" +) + +func main() { + if err := command.Execute(defaults.DefaultConfig()); err != nil { + os.Exit(1) + } +} diff --git a/services/clientlog/pkg/command/health.go b/services/clientlog/pkg/command/health.go new file mode 100644 index 0000000000..69e2b10ec2 --- /dev/null +++ b/services/clientlog/pkg/command/health.go @@ -0,0 +1,18 @@ +package command + +import ( + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "github.com/urfave/cli/v2" +) + +// Health is the entrypoint for the health command. +func Health(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "health", + Usage: "Check health status", + Action: func(c *cli.Context) error { + // Not implemented + return nil + }, + } +} diff --git a/services/clientlog/pkg/command/root.go b/services/clientlog/pkg/command/root.go new file mode 100644 index 0000000000..389caf1bb2 --- /dev/null +++ b/services/clientlog/pkg/command/root.go @@ -0,0 +1,34 @@ +package command + +import ( + "os" + + "github.com/owncloud/ocis/v2/ocis-pkg/clihelper" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "github.com/urfave/cli/v2" +) + +// GetCommands provides all commands for this service +func GetCommands(cfg *config.Config) cli.Commands { + return []*cli.Command{ + // start this service + Server(cfg), + + // interaction with this service + + // infos about this service + Health(cfg), + Version(cfg), + } +} + +// Execute is the entry point for the clientlog command. +func Execute(cfg *config.Config) error { + app := clihelper.DefaultApp(&cli.App{ + Name: "clientlog", + Usage: "starts clientlog service", + Commands: GetCommands(cfg), + }) + + return app.Run(os.Args) +} diff --git a/services/clientlog/pkg/command/server.go b/services/clientlog/pkg/command/server.go new file mode 100644 index 0000000000..b149a6ce27 --- /dev/null +++ b/services/clientlog/pkg/command/server.go @@ -0,0 +1,137 @@ +package command + +import ( + "context" + "fmt" + + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/events/stream" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/oklog/run" + "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/handlers" + "github.com/owncloud/ocis/v2/ocis-pkg/registry" + "github.com/owncloud/ocis/v2/ocis-pkg/service/debug" + "github.com/owncloud/ocis/v2/ocis-pkg/tracing" + "github.com/owncloud/ocis/v2/ocis-pkg/version" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config/parser" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/logging" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/metrics" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/service" + "github.com/urfave/cli/v2" +) + +// all events we care about +var _registeredEvents = []events.Unmarshaller{ + events.UploadReady{}, +} + +// Server is the entrypoint for the server command. +func Server(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "server", + Usage: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name), + Category: "server", + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + logger := logging.Configure(cfg.Service.Name, cfg.Log) + tracerProvider, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) + if err != nil { + return err + } + + /* + grpcClient, err := ogrpc.NewClient( + append(ogrpc.GetClientOptions(cfg.GRPCClientTLS), ogrpc.WithTraceProvider(tracerProvider))..., + ) + if err != nil { + return err + } + */ // TODO: remove + + gr := run.Group{} + ctx, cancel := func() (context.Context, context.CancelFunc) { + if cfg.Context == nil { + return context.WithCancel(context.Background()) + } + return context.WithCancel(cfg.Context) + }() + + mtrcs := metrics.New() + mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1) + + defer cancel() + + stream, err := stream.NatsFromConfig(cfg.Service.Name, stream.NatsConfig(cfg.Events)) + if err != nil { + return err + } + + tm, err := pool.StringToTLSMode(cfg.GRPCClientTLS.Mode) + if err != nil { + return err + } + gatewaySelector, err := pool.GatewaySelector( + cfg.RevaGateway, + pool.WithTLSCACert(cfg.GRPCClientTLS.CACert), + pool.WithTLSMode(tm), + pool.WithRegistry(registry.GetRegistry()), + pool.WithTracerProvider(tracerProvider), + ) + if err != nil { + return fmt.Errorf("could not get reva client selector: %s", err) + } + + { + svc, err := service.NewClientlogService( + service.Logger(logger), + service.Config(cfg), + service.Stream(stream), + service.GatewaySelector(gatewaySelector), + service.RegisteredEvents(_registeredEvents), + service.TraceProvider(tracerProvider), + ) + + if err != nil { + logger.Info().Err(err).Str("transport", "http").Msg("Failed to initialize server") + return err + } + + gr.Add(func() error { + return svc.Run() + }, func(err error) { + logger.Error(). + Str("transport", "http"). + Err(err). + Msg("Shutting down server") + + cancel() + }) + } + + { + server := debug.NewService( + debug.Logger(logger), + debug.Name(cfg.Service.Name), + debug.Version(version.GetString()), + debug.Address(cfg.Debug.Addr), + debug.Token(cfg.Debug.Token), + debug.Pprof(cfg.Debug.Pprof), + debug.Zpages(cfg.Debug.Zpages), + debug.Health(handlers.Health), + debug.Ready(handlers.Ready), + ) + + gr.Add(server.ListenAndServe, func(_ error) { + _ = server.Shutdown(ctx) + cancel() + }) + } + + return gr.Run() + }, + } +} diff --git a/services/clientlog/pkg/command/version.go b/services/clientlog/pkg/command/version.go new file mode 100644 index 0000000000..73c8776baa --- /dev/null +++ b/services/clientlog/pkg/command/version.go @@ -0,0 +1,19 @@ +package command + +import ( + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "github.com/urfave/cli/v2" +) + +// Version prints the service versions of all running instances. +func Version(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "version", + Usage: "print the version of this binary and the running service instances", + Category: "info", + Action: func(c *cli.Context) error { + // not implemented + return nil + }, + } +} diff --git a/services/clientlog/pkg/config/config.go b/services/clientlog/pkg/config/config.go new file mode 100644 index 0000000000..929950da90 --- /dev/null +++ b/services/clientlog/pkg/config/config.go @@ -0,0 +1,49 @@ +package config + +import ( + "context" + + "github.com/owncloud/ocis/v2/ocis-pkg/shared" +) + +// Config combines all available configuration parts. +type Config struct { + Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service + + Service Service `yaml:"-"` + + Tracing *Tracing `yaml:"tracing"` + Log *Log `yaml:"log"` + Debug Debug `yaml:"debug"` + + GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"` + + TokenManager *TokenManager `yaml:"token_manager"` + + RevaGateway string `yaml:"reva_gateway" env:"OCIS_REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata"` + Events Events `yaml:"events"` + + ServiceAccount ServiceAccount `yaml:"service_account"` + + Context context.Context `yaml:"-"` +} + +// Events combines the configuration options for the event bus. +type Events struct { + Endpoint string `yaml:"endpoint" env:"OCIS_EVENTS_ENDPOINT;CLIENTLOG_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture."` + Cluster string `yaml:"cluster" env:"OCIS_EVENTS_CLUSTER;CLIENTLOG_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system."` + TLSInsecure bool `yaml:"tls_insecure" env:"OCIS_INSECURE;CLIENTLOG_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates."` + TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OCIS_EVENTS_TLS_ROOT_CA_CERTIFICATE;CLIENTLOG_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided NOTIFICATIONS_EVENTS_TLS_INSECURE will be seen as false."` + EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS;CLIENTLOG_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services.."` +} + +// TokenManager is the config for using the reva token manager +type TokenManager struct { + JWTSecret string `yaml:"jwt_secret" env:"OCIS_JWT_SECRET;CLIENTLOG_JWT_SECRET" desc:"The secret to mint and validate jwt tokens."` +} + +// ServiceAccount is the configuration for the used service account +type ServiceAccount struct { + ServiceAccountID string `yaml:"service_account_id" env:"OCIS_SERVICE_ACCOUNT_ID;CLIENTLOG_SERVICE_ACCOUNT_ID" desc:"The ID of the service account the service should use. See the 'auth-service' service description for more details."` + ServiceAccountSecret string `yaml:"service_account_secret" env:"OCIS_SERVICE_ACCOUNT_SECRET;CLIENTLOG_SERVICE_ACCOUNT_SECRET" desc:"The service account secret."` +} diff --git a/services/clientlog/pkg/config/debug.go b/services/clientlog/pkg/config/debug.go new file mode 100644 index 0000000000..d29ac49163 --- /dev/null +++ b/services/clientlog/pkg/config/debug.go @@ -0,0 +1,9 @@ +package config + +// Debug defines the available debug configuration. +type Debug struct { + Addr string `yaml:"addr" env:"USERLOG_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed."` + Token string `yaml:"token" env:"USERLOG_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint."` + Pprof bool `yaml:"pprof" env:"USERLOG_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling."` + Zpages bool `yaml:"zpages" env:"USERLOG_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces."` +} diff --git a/services/clientlog/pkg/config/defaults/defaultconfig.go b/services/clientlog/pkg/config/defaults/defaultconfig.go new file mode 100644 index 0000000000..b0a2951bdc --- /dev/null +++ b/services/clientlog/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,80 @@ +package defaults + +import ( + "github.com/owncloud/ocis/v2/ocis-pkg/shared" + "github.com/owncloud/ocis/v2/ocis-pkg/structs" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" +) + +// FullDefaultConfig returns the full default config +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + EnsureDefaults(cfg) + Sanitize(cfg) + return cfg +} + +// DefaultConfig return the default configuration +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9260", + Token: "", + Pprof: false, + Zpages: false, + }, + Service: config.Service{ + Name: "clientlog", + }, + Events: config.Events{ + Endpoint: "127.0.0.1:9233", + Cluster: "ocis-cluster", + EnableTLS: false, + }, + RevaGateway: shared.DefaultRevaConfig().Address, + } +} + +// EnsureDefaults ensures the config contains default values +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for "envdecode". + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + + if cfg.GRPCClientTLS == nil && cfg.Commons != nil { + cfg.GRPCClientTLS = structs.CopyOrZeroValue(cfg.Commons.GRPCClientTLS) + } + + if cfg.TokenManager == nil && cfg.Commons != nil && cfg.Commons.TokenManager != nil { + cfg.TokenManager = &config.TokenManager{ + JWTSecret: cfg.Commons.TokenManager.JWTSecret, + } + } else if cfg.TokenManager == nil { + cfg.TokenManager = &config.TokenManager{} + } + + // provide with defaults for shared tracing, since we need a valid destination address for "envdecode". + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +// Sanitize sanitizes the config +func Sanitize(cfg *config.Config) { + // sanitize config +} diff --git a/services/clientlog/pkg/config/log.go b/services/clientlog/pkg/config/log.go new file mode 100644 index 0000000000..c3cc38911f --- /dev/null +++ b/services/clientlog/pkg/config/log.go @@ -0,0 +1,9 @@ +package config + +// Log defines the available log configuration. +type Log struct { + Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;USERLOG_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'."` + Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;USERLOG_LOG_PRETTY" desc:"Activates pretty log output."` + Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;USERLOG_LOG_COLOR" desc:"Activates colorized log output."` + File string `mapstructure:"file" env:"OCIS_LOG_FILE;USERLOG_LOG_FILE" desc:"The path to the log file. Activates logging to this file if set."` +} diff --git a/services/clientlog/pkg/config/parser/parse.go b/services/clientlog/pkg/config/parser/parse.go new file mode 100644 index 0000000000..99ad1d14cc --- /dev/null +++ b/services/clientlog/pkg/config/parser/parse.go @@ -0,0 +1,43 @@ +package parser + +import ( + "errors" + + ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis-pkg/shared" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config/defaults" + + "github.com/owncloud/ocis/v2/ocis-pkg/config/envdecode" +) + +// ParseConfig loads configuration from known paths. +func ParseConfig(cfg *config.Config) error { + _, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg) + if err != nil { + return err + } + + defaults.EnsureDefaults(cfg) + + // load all env variables relevant to the config in the current context. + if err := envdecode.Decode(cfg); err != nil { + // no environment variable set for this config is an expected "error" + if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) { + return err + } + } + + defaults.Sanitize(cfg) + + return Validate(cfg) +} + +// Validate validates the config +func Validate(cfg *config.Config) error { + if cfg.TokenManager.JWTSecret == "" { + return shared.MissingJWTTokenError(cfg.Service.Name) + } + + return nil +} diff --git a/services/clientlog/pkg/config/service.go b/services/clientlog/pkg/config/service.go new file mode 100644 index 0000000000..d1eac383f0 --- /dev/null +++ b/services/clientlog/pkg/config/service.go @@ -0,0 +1,6 @@ +package config + +// Service defines the available service configuration. +type Service struct { + Name string `yaml:"-"` +} diff --git a/services/clientlog/pkg/config/tracing.go b/services/clientlog/pkg/config/tracing.go new file mode 100644 index 0000000000..66f0332ec1 --- /dev/null +++ b/services/clientlog/pkg/config/tracing.go @@ -0,0 +1,21 @@ +package config + +import "github.com/owncloud/ocis/v2/ocis-pkg/tracing" + +// Tracing defines the available tracing configuration. +type Tracing struct { + Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;USERLOG_TRACING_ENABLED" desc:"Activates tracing."` + Type string `yaml:"type" env:"OCIS_TRACING_TYPE;USERLOG_TRACING_TYPE" desc:"The type of tracing. Defaults to '', which is the same as 'jaeger'. Allowed tracing types are 'jaeger' and '' as of now."` + Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;USERLOG_TRACING_ENDPOINT" desc:"The endpoint of the tracing agent."` + Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;USERLOG_TRACING_COLLECTOR" desc:"The HTTP endpoint for sending spans directly to a collector, i.e. http://jaeger-collector:14268/api/traces. Only used if the tracing endpoint is unset."` +} + +// Convert Tracing to the tracing package's Config struct. +func (t Tracing) Convert() tracing.Config { + return tracing.Config{ + Enabled: t.Enabled, + Type: t.Type, + Endpoint: t.Endpoint, + Collector: t.Collector, + } +} diff --git a/services/clientlog/pkg/logging/logging.go b/services/clientlog/pkg/logging/logging.go new file mode 100644 index 0000000000..516b592463 --- /dev/null +++ b/services/clientlog/pkg/logging/logging.go @@ -0,0 +1,17 @@ +package logging + +import ( + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" +) + +// LoggerFromConfig initializes a service-specific logger instance. +func Configure(name string, cfg *config.Log) log.Logger { + return log.NewLogger( + log.Name(name), + log.Level(cfg.Level), + log.Pretty(cfg.Pretty), + log.Color(cfg.Color), + log.File(cfg.File), + ) +} diff --git a/services/clientlog/pkg/metrics/metrics.go b/services/clientlog/pkg/metrics/metrics.go new file mode 100644 index 0000000000..c1f188f2ae --- /dev/null +++ b/services/clientlog/pkg/metrics/metrics.go @@ -0,0 +1,35 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +var ( + // Namespace defines the namespace for the defines metrics. + Namespace = "ocis" + + // Subsystem defines the subsystem for the defines metrics. + Subsystem = "clientlog" +) + +// Metrics defines the available metrics of this service. +type Metrics struct { + BuildInfo *prometheus.GaugeVec +} + +// New initializes the available metrics. +func New() *Metrics { + m := &Metrics{ + BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "build_info", + Help: "Build information", + }, []string{"version"}), + } + + _ = prometheus.Register( + m.BuildInfo, + ) + + // TODO: implement metrics + return m +} diff --git a/services/clientlog/pkg/service/options.go b/services/clientlog/pkg/service/options.go new file mode 100644 index 0000000000..26509a3ed0 --- /dev/null +++ b/services/clientlog/pkg/service/options.go @@ -0,0 +1,65 @@ +package service + +import ( + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "go.opentelemetry.io/otel/trace" +) + +// Option for the clientlog service +type Option func(*Options) + +// Options for the clientlog service +type Options struct { + Logger log.Logger + Stream events.Stream + Config *config.Config + GatewaySelector pool.Selectable[gateway.GatewayAPIClient] + RegisteredEvents []events.Unmarshaller + TraceProvider trace.TracerProvider +} + +// Logger configures a logger for the clientlog service +func Logger(log log.Logger) Option { + return func(o *Options) { + o.Logger = log + } +} + +// Stream configures an event stream for the clientlog service +func Stream(s events.Stream) Option { + return func(o *Options) { + o.Stream = s + } +} + +// Config adds the config for the clientlog service +func Config(c *config.Config) Option { + return func(o *Options) { + o.Config = c + } +} + +// GatewaySelector adds a grpc client selector for the gateway service +func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option { + return func(o *Options) { + o.GatewaySelector = gatewaySelector + } +} + +// RegisteredEvents registers the events the service should listen to +func RegisteredEvents(e []events.Unmarshaller) Option { + return func(o *Options) { + o.RegisteredEvents = e + } +} + +// TraceProvider adds a tracer provider for the clientlog service +func TraceProvider(tp trace.TracerProvider) Option { + return func(o *Options) { + o.TraceProvider = tp + } +} diff --git a/services/clientlog/pkg/service/service.go b/services/clientlog/pkg/service/service.go new file mode 100644 index 0000000000..4f9f6bdfd3 --- /dev/null +++ b/services/clientlog/pkg/service/service.go @@ -0,0 +1,134 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/v2/pkg/utils" + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "go.opentelemetry.io/otel/trace" +) + +// ClientlogService is the service responsible for user activities +type ClientlogService struct { + log log.Logger + cfg *config.Config + gatewaySelector pool.Selectable[gateway.GatewayAPIClient] + registeredEvents map[string]events.Unmarshaller // ? + tp trace.TracerProvider + tracer trace.Tracer + publisher events.Publisher + ch <-chan events.Event +} + +// NewClientlogService returns a clientlog service +func NewClientlogService(opts ...Option) (*ClientlogService, error) { + o := &Options{} + for _, opt := range opts { + opt(o) + } + + if o.Stream == nil { + return nil, fmt.Errorf("need non nil stream (%v) to work properly", o.Stream) + } + + ch, err := events.Consume(o.Stream, "clientlog", o.RegisteredEvents...) + if err != nil { + return nil, err + } + + cl := &ClientlogService{ + log: o.Logger, + cfg: o.Config, + gatewaySelector: o.GatewaySelector, + registeredEvents: make(map[string]events.Unmarshaller), + tp: o.TraceProvider, + tracer: o.TraceProvider.Tracer("github.com/owncloud/ocis/services/clientlog/pkg/service"), + publisher: o.Stream, + ch: ch, + } + + for _, e := range o.RegisteredEvents { + typ := reflect.TypeOf(e) + cl.registeredEvents[typ.String()] = e + } + + return cl, nil +} + +// Run runs the service +func (cl *ClientlogService) Run() error { + for event := range cl.ch { + cl.processEvent(event) + } + + return nil +} + +func (cl *ClientlogService) processEvent(event events.Event) { + gwc, err := cl.gatewaySelector.Next() + if err != nil { + cl.log.Error().Err(err).Interface("event", event).Msg("error getting gatway client") + return + } + + ctx, err := utils.GetServiceUserContext(cl.cfg.ServiceAccount.ServiceAccountID, gwc, cl.cfg.ServiceAccount.ServiceAccountSecret) + if err != nil { + cl.log.Error().Err(err).Interface("event", event).Msg("error authenticating service user") + return + } + + var users []string + switch e := event.Event.(type) { + default: + err = errors.New("unhandled event") + case events.UploadReady: + info, err := utils.GetResource(ctx, e.FileRef, gwc) + if err != nil { + cl.log.Error().Err(err).Interface("event", event).Msg("error getting resource") + return + } + + users, err = utils.GetSpaceMembers(ctx, info.GetSpace().GetId().GetOpaqueId(), gwc, utils.ViewerRole) + if err != nil { + cl.log.Error().Err(err).Interface("event", event).Msg("error getting space members") + return + } + } + + if err != nil { + cl.log.Info().Err(err).Interface("event", event).Msg("error gathering members for event") + return + } + + // II) instruct sse service to send the information + for _, id := range users { + if err := cl.sendSSE(id, event); err != nil { + cl.log.Error().Err(err).Str("userID", id).Str("eventid", event.ID).Msg("failed to store event for user") + return + } + } +} + +func (cl *ClientlogService) sendSSE(userid string, event events.Event) error { + // TODO: convert event + ev := event + + b, err := json.Marshal(ev) + if err != nil { + return err + } + + return events.Publish(context.Background(), cl.publisher, events.SendSSE{ + UserID: userid, + Type: "clientlog-notification", + Message: b, + }) +} diff --git a/services/clientlog/pkg/service/service_suit_test.go b/services/clientlog/pkg/service/service_suit_test.go new file mode 100644 index 0000000000..ab7fb7b198 --- /dev/null +++ b/services/clientlog/pkg/service/service_suit_test.go @@ -0,0 +1,26 @@ +package service_test + +import ( + "testing" + + "github.com/owncloud/ocis/v2/ocis-pkg/registry" + mRegistry "go-micro.dev/v4/registry" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func init() { + registry.Configure("memory") + r := registry.GetRegistry() + service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") + service.Nodes = []*mRegistry.Node{{ + Address: "any", + }} + + _ = r.Register(service) +} +func TestSearch(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Userlog service Suite") +} diff --git a/services/clientlog/pkg/service/service_test.go b/services/clientlog/pkg/service/service_test.go new file mode 100644 index 0000000000..922fe8e45c --- /dev/null +++ b/services/clientlog/pkg/service/service_test.go @@ -0,0 +1,3 @@ +package service_test + +// TODO: TEST! diff --git a/services/sse/README.md b/services/sse/README.md index ac5e43ae41..332d052dd0 100644 --- a/services/sse/README.md +++ b/services/sse/README.md @@ -2,6 +2,13 @@ The `sse` service is responsible for sending sse (Server-Sent Events) to a user. See [What is Server-Sent Events](https://medium.com/yemeksepeti-teknoloji/what-is-server-sent-events-sse-and-how-to-implement-it-904938bffd73) for a simple introduction and examples of server sent events. +## The `...log` service ecosystem + +`...log` services (`userlog`, `clientlog`) are responsible for composing notifications for a certain audience. + - `userlog` service translates and adjust messages to be human readable + - `clientlog` service composes machine readable messages so clients can act without needing to query the server + - `sse` service is only responsible for sending these messages. It does not care about their form or language + ## Subscribing Clients can subscribe to the `/sse` endpoint to be informed by the server when an event happens. The `sse` endpoint will respect language changes of the user without needing to reconnect. Note that SSE has a limitation of six open connections per browser which can be reached if one has opened various tabs of the Web UI pointing to the same Infinite Scale instance. diff --git a/services/userlog/README.md b/services/userlog/README.md index aaaf18f409..e89e1dbc5f 100644 --- a/services/userlog/README.md +++ b/services/userlog/README.md @@ -2,6 +2,13 @@ The `userlog` service is a mediator between the `eventhistory` service and clients who want to be informed about user related events. It provides an API to retrieve those. +## The `...log` service ecosystem + +`...log` services (`userlog`, `clientlog`) are responsible for composing notifications for a certain audience. + - `userlog` service translates and adjust messages to be human readable + - `clientlog` service composes machine readable messages so clients can act without needing to query the server + - `sse` service is only responsible for sending these messages. It does not care about their form or language + ## Prerequisites Running the `userlog` service without running the `eventhistory` service is not possible. From 74f4143f0ba8525534fc87f13d60e2933f4a56d5 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 5 Sep 2023 15:03:10 +0200 Subject: [PATCH 2/6] use new utils methods in userlog Signed-off-by: jkoberg --- services/userlog/pkg/config/config.go | 9 +- .../pkg/config/defaults/defaultconfig.go | 4 - services/userlog/pkg/config/parser/parse.go | 4 - services/userlog/pkg/service/conversion.go | 74 ++--- services/userlog/pkg/service/http.go | 17 +- services/userlog/pkg/service/service.go | 276 ++---------------- 6 files changed, 70 insertions(+), 314 deletions(-) diff --git a/services/userlog/pkg/config/config.go b/services/userlog/pkg/config/config.go index 0779c259a8..a2b71d9823 100644 --- a/services/userlog/pkg/config/config.go +++ b/services/userlog/pkg/config/config.go @@ -22,11 +22,10 @@ type Config struct { TokenManager *TokenManager `yaml:"token_manager"` - MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;USERLOG_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary to access resources from other services."` - RevaGateway string `yaml:"reva_gateway" env:"OCIS_REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata"` - TranslationPath string `yaml:"translation_path" env:"OCIS_TRANSLATION_PATH;USERLOG_TRANSLATION_PATH" desc:"(optional) Set this to a path with custom translations to overwrite the builtin translations. Note that file and folder naming rules apply, see the documentation for more details."` - Events Events `yaml:"events"` - Persistence Persistence `yaml:"persistence"` + RevaGateway string `yaml:"reva_gateway" env:"OCIS_REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata"` + TranslationPath string `yaml:"translation_path" env:"OCIS_TRANSLATION_PATH;USERLOG_TRANSLATION_PATH" desc:"(optional) Set this to a path with custom translations to overwrite the builtin translations. Note that file and folder naming rules apply, see the documentation for more details."` + Events Events `yaml:"events"` + Persistence Persistence `yaml:"persistence"` DisableSSE bool `yaml:"disable_sse" env:"OCIS_DISABLE_SSE,USERLOG_DISABLE_SSE" desc:"Disables server-sent events (sse). When disabled, clients will no longer receive sse notifications."` diff --git a/services/userlog/pkg/config/defaults/defaultconfig.go b/services/userlog/pkg/config/defaults/defaultconfig.go index 1dfd6318ca..6b174238a8 100644 --- a/services/userlog/pkg/config/defaults/defaultconfig.go +++ b/services/userlog/pkg/config/defaults/defaultconfig.go @@ -69,10 +69,6 @@ func EnsureDefaults(cfg *config.Config) { cfg.Log = &config.Log{} } - if cfg.MachineAuthAPIKey == "" && cfg.Commons != nil && cfg.Commons.MachineAuthAPIKey != "" { - cfg.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey - } - if cfg.GRPCClientTLS == nil && cfg.Commons != nil { cfg.GRPCClientTLS = structs.CopyOrZeroValue(cfg.Commons.GRPCClientTLS) } diff --git a/services/userlog/pkg/config/parser/parse.go b/services/userlog/pkg/config/parser/parse.go index 11340059cc..64e69ac91a 100644 --- a/services/userlog/pkg/config/parser/parse.go +++ b/services/userlog/pkg/config/parser/parse.go @@ -35,10 +35,6 @@ func ParseConfig(cfg *config.Config) error { // Validate validates the config func Validate(cfg *config.Config) error { - if cfg.MachineAuthAPIKey == "" { - return shared.MissingMachineAuthApiKeyError(cfg.Service.Name) - } - if cfg.TokenManager.JWTSecret == "" { return shared.MissingJWTTokenError(cfg.Service.Name) } diff --git a/services/userlog/pkg/service/conversion.go b/services/userlog/pkg/service/conversion.go index c589753fc8..c3cfc70437 100644 --- a/services/userlog/pkg/service/conversion.go +++ b/services/userlog/pkg/service/conversion.go @@ -16,7 +16,6 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/events" - "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/utils" "github.com/leonelquinteros/gotext" @@ -52,32 +51,29 @@ type OC10Notification struct { // Converter is responsible for converting eventhistory events to OC10Notifications type Converter struct { - locale string - gatewaySelector pool.Selectable[gateway.GatewayAPIClient] - serviceAccountID string - serviceAccountSecret string - serviceName string - translationPath string + locale string + gwc gateway.GatewayAPIClient + serviceName string + translationPath string + serviceAccountContext context.Context // cached within one request not to query other service too much - spaces map[string]*storageprovider.StorageSpace - users map[string]*user.User - resources map[string]*storageprovider.ResourceInfo - serviceAccountContext context.Context + spaces map[string]*storageprovider.StorageSpace + users map[string]*user.User + resources map[string]*storageprovider.ResourceInfo } // NewConverter returns a new Converter -func NewConverter(loc string, gatewaySelector pool.Selectable[gateway.GatewayAPIClient], machineAuthAPIKey string, name string, translationPath string, serviceAccountID string, serviceAccountSecret string) *Converter { +func NewConverter(ctx context.Context, loc string, gwc gateway.GatewayAPIClient, name string, translationPath string) *Converter { return &Converter{ - locale: loc, - gatewaySelector: gatewaySelector, - serviceAccountID: serviceAccountID, - serviceAccountSecret: serviceAccountSecret, - serviceName: name, - translationPath: translationPath, - spaces: make(map[string]*storageprovider.StorageSpace), - users: make(map[string]*user.User), - resources: make(map[string]*storageprovider.ResourceInfo), + locale: loc, + gwc: gwc, + serviceName: name, + translationPath: translationPath, + serviceAccountContext: ctx, + spaces: make(map[string]*storageprovider.StorageSpace), + users: make(map[string]*user.User), + resources: make(map[string]*storageprovider.ResourceInfo), } } @@ -172,12 +168,7 @@ func (c *Converter) spaceMessage(eventid string, nt NotificationTemplate, execut return OC10Notification{}, err } - ctx, err := c.authenticate() - if err != nil { - return OC10Notification{}, err - } - - space, err := c.getSpace(ctx, spaceid) + space, err := c.getSpace(c.serviceAccountContext, spaceid) if err != nil { return OC10Notification{}, err } @@ -211,12 +202,7 @@ func (c *Converter) shareMessage(eventid string, nt NotificationTemplate, execut return OC10Notification{}, err } - ctx, err := c.authenticate() - if err != nil { - return OC10Notification{}, err - } - - info, err := c.getResource(ctx, resourceid) + info, err := c.getResource(c.serviceAccountContext, resourceid) if err != nil { return OC10Notification{}, err } @@ -328,27 +314,11 @@ func (c *Converter) deprovisionMessage(nt NotificationTemplate, deproDate string }, nil } -func (c *Converter) authenticate() (context.Context, error) { - if c.serviceAccountContext != nil { - return c.serviceAccountContext, nil - } - - gatewayClient, err := c.gatewaySelector.Next() - if err != nil { - return nil, err - } - ctx, err := utils.GetServiceUserContext(c.serviceAccountID, gatewayClient, c.serviceAccountSecret) - if err == nil { - c.serviceAccountContext = ctx - } - return ctx, err -} - func (c *Converter) getSpace(ctx context.Context, spaceID string) (*storageprovider.StorageSpace, error) { if space, ok := c.spaces[spaceID]; ok { return space, nil } - space, err := getSpace(ctx, spaceID, c.gatewaySelector) + space, err := utils.GetSpace(ctx, spaceID, c.gwc) if err == nil { c.spaces[spaceID] = space } @@ -359,7 +329,7 @@ func (c *Converter) getResource(ctx context.Context, resourceID *storageprovider if r, ok := c.resources[resourceID.GetOpaqueId()]; ok { return r, nil } - resource, err := getResource(ctx, resourceID, c.gatewaySelector) + resource, err := utils.GetResourceByID(ctx, resourceID, c.gwc) if err == nil { c.resources[resourceID.GetOpaqueId()] = resource } @@ -370,7 +340,7 @@ func (c *Converter) getUser(ctx context.Context, userID *user.UserId) (*user.Use if u, ok := c.users[userID.GetOpaqueId()]; ok { return u, nil } - usr, err := getUser(ctx, userID, c.gatewaySelector) + usr, err := utils.GetUser(userID, c.gwc) if err == nil { c.users[userID.GetOpaqueId()] = usr } diff --git a/services/userlog/pkg/service/http.go b/services/userlog/pkg/service/http.go index 74167e4219..0d13f1c810 100644 --- a/services/userlog/pkg/service/http.go +++ b/services/userlog/pkg/service/http.go @@ -8,6 +8,7 @@ import ( "github.com/cs3org/reva/v2/pkg/appctx" revactx "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/cs3org/reva/v2/pkg/utils" "github.com/owncloud/ocis/v2/ocis-pkg/roles" "github.com/owncloud/ocis/v2/services/graph/pkg/service/v0/errorcode" settings "github.com/owncloud/ocis/v2/services/settings/pkg/service/v0" @@ -44,7 +45,21 @@ func (ul *UserlogService) HandleGetEvents(w http.ResponseWriter, r *http.Request Value: attribute.IntValue(len(evs)), }) - conv := ul.getConverter(r.Header.Get(HeaderAcceptLanguage)) + gwc, err := ul.gatewaySelector.Next() + if err != nil { + ul.log.Error().Err(err).Msg("cant get gateway client") + w.WriteHeader(http.StatusInternalServerError) + return + } + + ctx, err = utils.GetServiceUserContext(ul.cfg.ServiceAccount.ServiceAccountID, gwc, ul.cfg.ServiceAccount.ServiceAccountSecret) + if err != nil { + ul.log.Error().Err(err).Msg("cant get service account") + w.WriteHeader(http.StatusInternalServerError) + return + } + + conv := NewConverter(ctx, r.Header.Get(HeaderAcceptLanguage), gwc, ul.cfg.Service.Name, ul.cfg.TranslationPath) resp := GetEventResponseOC10{} for _, e := range evs { diff --git a/services/userlog/pkg/service/service.go b/services/userlog/pkg/service/service.go index a9d18ce44c..6e5897e9da 100644 --- a/services/userlog/pkg/service/service.go +++ b/services/userlog/pkg/service/service.go @@ -9,10 +9,7 @@ import ( "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/v2/pkg/utils" @@ -114,6 +111,18 @@ func (ul *UserlogService) processEvent(event events.Event) { err error ) + gwc, err := ul.gatewaySelector.Next() + if err != nil { + ul.log.Error().Err(err).Msg("cannot get gateway client") + return + } + + ctx, err := utils.GetServiceUserContext(ul.cfg.ServiceAccount.ServiceAccountID, gwc, ul.cfg.ServiceAccount.ServiceAccountSecret) + if err != nil { + ul.log.Error().Err(err).Msg("cannot get service account") + return + } + switch e := event.Event.(type) { default: err = errors.New("unhandled event") @@ -140,7 +149,7 @@ func (ul *UserlogService) processEvent(event events.Event) { // space related // TODO: how to find spaceadmins? case events.SpaceDisabled: executant = e.Executant - users, err = ul.findSpaceMembers(ul.mustAuthenticate(), e.ID.GetOpaqueId(), viewer) + users, err = utils.GetSpaceMembers(ctx, e.ID.GetOpaqueId(), gwc, utils.ViewerRole) case events.SpaceDeleted: executant = e.Executant for u := range e.FinalMembers { @@ -148,22 +157,22 @@ func (ul *UserlogService) processEvent(event events.Event) { } case events.SpaceShared: executant = e.Executant - users, err = ul.resolveID(ul.mustAuthenticate(), e.GranteeUserID, e.GranteeGroupID) + users, err = utils.ResolveID(ctx, e.GranteeUserID, e.GranteeGroupID, gwc) case events.SpaceUnshared: executant = e.Executant - users, err = ul.resolveID(ul.mustAuthenticate(), e.GranteeUserID, e.GranteeGroupID) + users, err = utils.ResolveID(ctx, e.GranteeUserID, e.GranteeGroupID, gwc) case events.SpaceMembershipExpired: - users, err = ul.resolveID(ul.mustAuthenticate(), e.GranteeUserID, e.GranteeGroupID) + users, err = utils.ResolveID(ctx, e.GranteeUserID, e.GranteeGroupID, gwc) // share related case events.ShareCreated: executant = e.Executant - users, err = ul.resolveID(ul.mustAuthenticate(), e.GranteeUserID, e.GranteeGroupID) + users, err = utils.ResolveID(ctx, e.GranteeUserID, e.GranteeGroupID, gwc) case events.ShareRemoved: executant = e.Executant - users, err = ul.resolveID(ul.mustAuthenticate(), e.GranteeUserID, e.GranteeGroupID) + users, err = utils.ResolveID(ctx, e.GranteeUserID, e.GranteeGroupID, gwc) case events.ShareExpired: - users, err = ul.resolveID(ul.mustAuthenticate(), e.GranteeUserID, e.GranteeGroupID) + users, err = utils.ResolveID(ctx, e.GranteeUserID, e.GranteeGroupID, gwc) } if err != nil { @@ -180,7 +189,12 @@ func (ul *UserlogService) processEvent(event events.Event) { // III) store the eventID for each user for _, id := range users { - if err := ul.addEventToUser(id, event); err != nil { + if !ul.cfg.DisableSSE { + if err := ul.sendSSE(ctx, id, event, gwc); err != nil { + ul.log.Error().Err(err).Str("userid", id).Str("eventid", event.ID).Msg("cannot create sse event") + } + } + if err := ul.addEventToUser(ctx, id, event); err != nil { ul.log.Error().Err(err).Str("userID", id).Str("eventid", event.ID).Msg("failed to store event for user") return } @@ -316,19 +330,14 @@ func (ul *UserlogService) DeleteGlobalEvents(ctx context.Context, evnames []stri }) } -func (ul *UserlogService) addEventToUser(userid string, event events.Event) error { - if !ul.cfg.DisableSSE { - if err := ul.sendSSE(userid, event); err != nil { - ul.log.Error().Err(err).Str("userid", userid).Str("eventid", event.ID).Msg("cannot create sse event") - } - } +func (ul *UserlogService) addEventToUser(ctx context.Context, userid string, event events.Event) error { return ul.alterUserEventList(userid, func(ids []string) []string { return append(ids, event.ID) }) } -func (ul *UserlogService) sendSSE(userid string, event events.Event) error { - ev, err := ul.getConverter(ul.getUserLocale(userid)).ConvertEvent(event.ID, event.Event) +func (ul *UserlogService) sendSSE(ctx context.Context, userid string, event events.Event, gwc gateway.GatewayAPIClient) error { + ev, err := NewConverter(ctx, ul.getUserLocale(userid), gwc, ul.cfg.Service.Name, ul.cfg.TranslationPath).ConvertEvent(event.ID, event.Event) if err != nil { return err } @@ -419,105 +428,6 @@ func (ul *UserlogService) alterGlobalEvents(ctx context.Context, alter func(map[ }) } -// we need the spaceid to inform other space members -// we need an owner to query space members -// we need to check the user has the required role to see the event -func (ul *UserlogService) findSpaceMembers(ctx context.Context, spaceID string, requiredRole permissionChecker) ([]string, error) { - if ctx == nil { - return nil, errors.New("need authenticated context to find space members") - } - - space, err := getSpace(ctx, spaceID, ul.gatewaySelector) - if err != nil { - return nil, err - } - - var users []string - switch space.SpaceType { - case "personal": - users = []string{space.GetOwner().GetId().GetOpaqueId()} - case "project": - if users, err = ul.gatherSpaceMembers(ctx, space, requiredRole); err != nil { - return nil, err - } - default: - // TODO: shares? other space types? - return nil, fmt.Errorf("unsupported space type: %s", space.SpaceType) - } - - return users, nil -} - -func (ul *UserlogService) gatherSpaceMembers(ctx context.Context, space *storageprovider.StorageSpace, hasRequiredRole permissionChecker) ([]string, error) { - var permissionsMap map[string]*storageprovider.ResourcePermissions - if err := utils.ReadJSONFromOpaque(space.GetOpaque(), "grants", &permissionsMap); err != nil { - return nil, err - } - - groupsMap := make(map[string]struct{}) - if opaqueGroups, ok := space.Opaque.Map["groups"]; ok { - _ = json.Unmarshal(opaqueGroups.GetValue(), &groupsMap) - } - - // we use a map to avoid duplicates - usermap := make(map[string]struct{}) - for id, perm := range permissionsMap { - if !hasRequiredRole(perm) { - // not allowed to receive event - continue - } - - if _, isGroup := groupsMap[id]; !isGroup { - usermap[id] = struct{}{} - continue - } - - usrs, err := ul.resolveGroup(ctx, id) - if err != nil { - ul.log.Error().Err(err).Str("groupID", id).Msg("failed to resolve group") - continue - } - - for _, u := range usrs { - usermap[u] = struct{}{} - } - } - - var users []string - for id := range usermap { - users = append(users, id) - } - - return users, nil -} - -func (ul *UserlogService) resolveID(ctx context.Context, userid *user.UserId, groupid *group.GroupId) ([]string, error) { - if userid != nil { - return []string{userid.GetOpaqueId()}, nil - } - - if ctx == nil { - return nil, errors.New("need ctx to resolve group id") - } - - return ul.resolveGroup(ctx, groupid.GetOpaqueId()) -} - -// resolves the users of a group -func (ul *UserlogService) resolveGroup(ctx context.Context, groupID string) ([]string, error) { - grp, err := getGroup(ctx, groupID, ul.gatewaySelector) - if err != nil { - return nil, err - } - - var userIDs []string - for _, m := range grp.GetMembers() { - userIDs = append(userIDs, m.GetOpaqueId()) - } - - return userIDs, nil -} - func (ul *UserlogService) getUserLocale(userid string) string { resp, err := ul.valueClient.GetValueByUniqueIdentifiers( micrometadata.Set(context.Background(), middleware.AccountID, userid), @@ -537,122 +447,6 @@ func (ul *UserlogService) getUserLocale(userid string) string { return val[0].GetStringValue() } -func (ul *UserlogService) getConverter(locale string) *Converter { - return NewConverter(locale, ul.gatewaySelector, ul.cfg.MachineAuthAPIKey, ul.cfg.Service.Name, ul.cfg.TranslationPath, ul.cfg.ServiceAccount.ServiceAccountID, ul.cfg.ServiceAccount.ServiceAccountSecret) -} - -func (ul *UserlogService) mustAuthenticate() context.Context { - ctx, err := authenticate(ul.cfg.ServiceAccount.ServiceAccountID, ul.gatewaySelector, ul.cfg.ServiceAccount.ServiceAccountSecret) - if err != nil { - ul.log.Error().Err(err).Str("accountid", ul.cfg.ServiceAccount.ServiceAccountID).Msg("failed to impersonate service account") - return nil - } - return ctx -} - -func authenticate(serviceAccountID string, gatewaySelector pool.Selectable[gateway.GatewayAPIClient], serviceAccountSecret string) (context.Context, error) { - gatewayClient, err := gatewaySelector.Next() - if err != nil { - return nil, err - } - - return utils.GetServiceUserContext(serviceAccountID, gatewayClient, serviceAccountSecret) -} - -func getSpace(ctx context.Context, spaceID string, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*storageprovider.StorageSpace, error) { - gatewayClient, err := gatewaySelector.Next() - if err != nil { - return nil, err - } - - res, err := gatewayClient.ListStorageSpaces(ctx, listStorageSpaceRequest(spaceID)) - if err != nil { - return nil, err - } - - if res.GetStatus().GetCode() != rpc.Code_CODE_OK { - return nil, fmt.Errorf("error while getting space: (%v) %s", res.GetStatus().GetCode(), res.GetStatus().GetMessage()) - } - - if len(res.StorageSpaces) == 0 { - return nil, fmt.Errorf("error getting storage space %s: no space returned", spaceID) - } - - return res.StorageSpaces[0], nil -} - -func getUser(ctx context.Context, userid *user.UserId, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*user.User, error) { - gatewayClient, err := gatewaySelector.Next() - if err != nil { - return nil, err - } - - getUserResponse, err := gatewayClient.GetUser(context.Background(), &user.GetUserRequest{ - UserId: userid, - }) - if err != nil { - return nil, err - } - - if getUserResponse.Status.Code != rpc.Code_CODE_OK { - return nil, fmt.Errorf("error getting user: %s", getUserResponse.Status.Message) - } - - return getUserResponse.GetUser(), nil -} - -func getGroup(ctx context.Context, groupid string, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*group.Group, error) { - gatewayClient, err := gatewaySelector.Next() - if err != nil { - return nil, err - } - - r, err := gatewayClient.GetGroup(ctx, &group.GetGroupRequest{GroupId: &group.GroupId{OpaqueId: groupid}}) - if err != nil { - return nil, err - } - - if r.GetStatus().GetCode() != rpc.Code_CODE_OK { - return nil, fmt.Errorf("unexpected status code from gateway client: %d", r.GetStatus().GetCode()) - } - - return r.GetGroup(), nil -} - -func getResource(ctx context.Context, resourceid *storageprovider.ResourceId, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*storageprovider.ResourceInfo, error) { - gatewayClient, err := gatewaySelector.Next() - if err != nil { - return nil, err - } - - res, err := gatewayClient.Stat(ctx, &storageprovider.StatRequest{Ref: &storageprovider.Reference{ResourceId: resourceid}}) - if err != nil { - return nil, err - } - - if res.GetStatus().GetCode() != rpc.Code_CODE_OK { - return nil, fmt.Errorf("unexpected status code while getting space: %v", res.GetStatus().GetCode()) - } - - return res.GetInfo(), nil -} - -func listStorageSpaceRequest(spaceID string) *storageprovider.ListStorageSpacesRequest { - return &storageprovider.ListStorageSpacesRequest{ - Opaque: utils.AppendPlainToOpaque(nil, "unrestricted", "true"), - Filters: []*storageprovider.ListStorageSpacesRequest_Filter{ - { - Type: storageprovider.ListStorageSpacesRequest_Filter_TYPE_ID, - Term: &storageprovider.ListStorageSpacesRequest_Filter_Id{ - Id: &storageprovider.StorageSpaceId{ - OpaqueId: spaceID, - }, - }, - }, - }, - } -} - func removeExecutant(users []string, executant *user.UserId) []string { var usrs []string for _, u := range users { @@ -662,17 +456,3 @@ func removeExecutant(users []string, executant *user.UserId) []string { } return usrs } - -type permissionChecker func(*storageprovider.ResourcePermissions) bool - -func viewer(perms *storageprovider.ResourcePermissions) bool { - return perms.Stat -} - -func editor(perms *storageprovider.ResourcePermissions) bool { - return perms.InitiateFileUpload -} - -func manager(perms *storageprovider.ResourcePermissions) bool { - return perms.DenyGrant -} From da3f975d8736736b20faf83bfd9001d13ec0aba8 Mon Sep 17 00:00:00 2001 From: kobergj Date: Wed, 6 Sep 2023 10:42:47 +0200 Subject: [PATCH 3/6] improve clientlog documentation Co-authored-by: Martin --- services/clientlog/README.md | 18 +++++++++--------- services/sse/README.md | 10 +++++----- services/userlog/README.md | 10 +++++----- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/services/clientlog/README.md b/services/clientlog/README.md index 9421f5b3bd..b00bccc8e4 100644 --- a/services/clientlog/README.md +++ b/services/clientlog/README.md @@ -1,14 +1,14 @@ -# Clientlog service +# Clientlog Service -The `clientlog` service is responsible for composing machine readable notifications for clients +The `clientlog` service is responsible for composing machine readable notifications for clients. Clients are apps and web interfaces. -## The `...log` service ecosystem +## The Log Service Ecosystem -`...log` services (`userlog`, `clientlog`) are responsible for composing notifications for a certain audience. - - `userlog` service translates and adjust messages to be human readable - - `clientlog` service composes machine readable messages so clients can act without needing to query the server - - `sse` service is only responsible for sending these messages. It does not care about their form or language +Log services like the `userlog`, `clientlog` and `sse` are responsible for composing notifications for a certain audience. + - The `userlog` service translates and adjusts messages to be human readable. + - The `clientlog` service composes machine readable messages, so clients can act without the need to query the server. + - The `sse` service is only responsible for sending these messages. It does not care about their form or language. -## `clientlog` events +## Clientlog Events -The messages the `clientlog` service sends are meant to be used by clients, not by users. The client might for example be informed that a file is finished postprocessing, so it can make the file available to the user without needing to make another call to the server. +The messages the `clientlog` service sends are intended for the use by clients, not by users. The client might for example be informed that a file has finished post-processing. With that, the client can make the file available to the user without additional server queries. diff --git a/services/sse/README.md b/services/sse/README.md index 332d052dd0..11e47b7c9e 100644 --- a/services/sse/README.md +++ b/services/sse/README.md @@ -2,12 +2,12 @@ The `sse` service is responsible for sending sse (Server-Sent Events) to a user. See [What is Server-Sent Events](https://medium.com/yemeksepeti-teknoloji/what-is-server-sent-events-sse-and-how-to-implement-it-904938bffd73) for a simple introduction and examples of server sent events. -## The `...log` service ecosystem +## The Log Service Ecosystem -`...log` services (`userlog`, `clientlog`) are responsible for composing notifications for a certain audience. - - `userlog` service translates and adjust messages to be human readable - - `clientlog` service composes machine readable messages so clients can act without needing to query the server - - `sse` service is only responsible for sending these messages. It does not care about their form or language +Log services like the `userlog`, `clientlog` and `sse` are responsible for composing notifications for a certain audience. + - The `userlog` service translates and adjusts messages to be human readable. + - The `clientlog` service composes machine readable messages, so clients can act without the need to query the server. + - The `sse` service is only responsible for sending these messages. It does not care about their form or language. ## Subscribing diff --git a/services/userlog/README.md b/services/userlog/README.md index e89e1dbc5f..0261b2ceea 100644 --- a/services/userlog/README.md +++ b/services/userlog/README.md @@ -2,12 +2,12 @@ The `userlog` service is a mediator between the `eventhistory` service and clients who want to be informed about user related events. It provides an API to retrieve those. -## The `...log` service ecosystem +## The Log Service Ecosystem -`...log` services (`userlog`, `clientlog`) are responsible for composing notifications for a certain audience. - - `userlog` service translates and adjust messages to be human readable - - `clientlog` service composes machine readable messages so clients can act without needing to query the server - - `sse` service is only responsible for sending these messages. It does not care about their form or language +Log services like the `userlog`, `clientlog` and `sse` are responsible for composing notifications for a certain audience. + - The `userlog` service translates and adjusts messages to be human readable. + - The `clientlog` service composes machine readable messages, so clients can act without the need to query the server. + - The `sse` service is only responsible for sending these messages. It does not care about their form or language. ## Prerequisites From 35f2cd685a41fbbb5db08e5588596f55299874b4 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 6 Sep 2023 11:33:14 +0200 Subject: [PATCH 4/6] convert event to common type Signed-off-by: jkoberg --- services/clientlog/pkg/config/tracing.go | 8 +++---- services/clientlog/pkg/service/service.go | 28 ++++++++++++++--------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/services/clientlog/pkg/config/tracing.go b/services/clientlog/pkg/config/tracing.go index 66f0332ec1..dd041c6325 100644 --- a/services/clientlog/pkg/config/tracing.go +++ b/services/clientlog/pkg/config/tracing.go @@ -4,10 +4,10 @@ import "github.com/owncloud/ocis/v2/ocis-pkg/tracing" // Tracing defines the available tracing configuration. type Tracing struct { - Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;USERLOG_TRACING_ENABLED" desc:"Activates tracing."` - Type string `yaml:"type" env:"OCIS_TRACING_TYPE;USERLOG_TRACING_TYPE" desc:"The type of tracing. Defaults to '', which is the same as 'jaeger'. Allowed tracing types are 'jaeger' and '' as of now."` - Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;USERLOG_TRACING_ENDPOINT" desc:"The endpoint of the tracing agent."` - Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;USERLOG_TRACING_COLLECTOR" desc:"The HTTP endpoint for sending spans directly to a collector, i.e. http://jaeger-collector:14268/api/traces. Only used if the tracing endpoint is unset."` + Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;CLIENTLOG_TRACING_ENABLED" desc:"Activates tracing."` + Type string `yaml:"type" env:"OCIS_TRACING_TYPE;CLIENTLOG_TRACING_TYPE" desc:"The type of tracing. Defaults to '', which is the same as 'jaeger'. Allowed tracing types are 'jaeger' and '' as of now."` + Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;CLIENTLOG_TRACING_ENDPOINT" desc:"The endpoint of the tracing agent."` + Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;CLIENTLOG_TRACING_COLLECTOR" desc:"The HTTP endpoint for sending spans directly to a collector, i.e. http://jaeger-collector:14268/api/traces. Only used if the tracing endpoint is unset."` } // Convert Tracing to the tracing package's Config struct. diff --git a/services/clientlog/pkg/service/service.go b/services/clientlog/pkg/service/service.go index 4f9f6bdfd3..f382c7b972 100644 --- a/services/clientlog/pkg/service/service.go +++ b/services/clientlog/pkg/service/service.go @@ -10,12 +10,19 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/utils" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" "go.opentelemetry.io/otel/trace" ) +// ClientNotification is the event the clientlog service is sending to the client +type ClientNotification struct { + Type string + ItemID string +} + // ClientlogService is the service responsible for user activities type ClientlogService struct { log log.Logger @@ -85,7 +92,10 @@ func (cl *ClientlogService) processEvent(event events.Event) { return } - var users []string + var ( + users []string + noti ClientNotification + ) switch e := event.Event.(type) { default: err = errors.New("unhandled event") @@ -96,11 +106,10 @@ func (cl *ClientlogService) processEvent(event events.Event) { return } + noti.Type = "postprocessing-finished" + noti.ItemID = storagespace.FormatResourceID(*info.GetId()) + users, err = utils.GetSpaceMembers(ctx, info.GetSpace().GetId().GetOpaqueId(), gwc, utils.ViewerRole) - if err != nil { - cl.log.Error().Err(err).Interface("event", event).Msg("error getting space members") - return - } } if err != nil { @@ -110,18 +119,15 @@ func (cl *ClientlogService) processEvent(event events.Event) { // II) instruct sse service to send the information for _, id := range users { - if err := cl.sendSSE(id, event); err != nil { + if err := cl.sendSSE(id, noti); err != nil { cl.log.Error().Err(err).Str("userID", id).Str("eventid", event.ID).Msg("failed to store event for user") return } } } -func (cl *ClientlogService) sendSSE(userid string, event events.Event) error { - // TODO: convert event - ev := event - - b, err := json.Marshal(ev) +func (cl *ClientlogService) sendSSE(userid string, noti ClientNotification) error { + b, err := json.Marshal(noti) if err != nil { return err } From 6df7f727e6730f86a20760474f548dd548ca23aa Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 5 Sep 2023 10:25:00 +0200 Subject: [PATCH 5/6] bump reva Signed-off-by: jkoberg --- changelog/unreleased/bump-reva.md | 1 + go.mod | 2 +- go.sum | 4 +- .../owncloud/ocs/data/capabilities.go | 26 ++- .../handlers/apps/sharing/shares/public.go | 21 +- .../handlers/apps/sharing/shares/shares.go | 45 +++- .../http/services/owncloud/ocs/ocs.go | 5 +- .../reva/v2/pkg/password/password_policies.go | 157 ++++++++++++++ .../cs3org/reva/v2/pkg/utils/grpc.go | 196 +++++++++++++++++- vendor/modules.txt | 3 +- 10 files changed, 430 insertions(+), 30 deletions(-) create mode 100644 vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go diff --git a/changelog/unreleased/bump-reva.md b/changelog/unreleased/bump-reva.md index 488c8b99ba..5052a056b6 100644 --- a/changelog/unreleased/bump-reva.md +++ b/changelog/unreleased/bump-reva.md @@ -5,3 +5,4 @@ bumps reva version https://github.com/owncloud/ocis/pull/7138 https://github.com/owncloud/ocis/pull/6427 https://github.com/owncloud/ocis/pull/7178 +https://github.com/owncloud/ocis/pull/7217 diff --git a/go.mod b/go.mod index 6f285d7544..1f04762a84 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.6.0 github.com/cs3org/go-cs3apis v0.0.0-20230516150832-730ac860c71d - github.com/cs3org/reva/v2 v2.16.1-0.20230904124812-2ebd3e92cdb9 + github.com/cs3org/reva/v2 v2.16.1-0.20230906142214-864d9012e37f github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 diff --git a/go.sum b/go.sum index 28884086df..7eb0df9a32 100644 --- a/go.sum +++ b/go.sum @@ -1013,8 +1013,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4= github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc= github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA= -github.com/cs3org/reva/v2 v2.16.1-0.20230904124812-2ebd3e92cdb9 h1:YWkkoagYryRH56z5pw04Py0Ebg7spCZaWpsxjwGiNgU= -github.com/cs3org/reva/v2 v2.16.1-0.20230904124812-2ebd3e92cdb9/go.mod h1:RvhuweTFqzezjUFU0SIdTXakrEx9vJlMvQ7znPXSP1g= +github.com/cs3org/reva/v2 v2.16.1-0.20230906142214-864d9012e37f h1:0mbvh+AvpYOp29R5LFgeqddyI0uKZnO/E3MyPEIaYdg= +github.com/cs3org/reva/v2 v2.16.1-0.20230906142214-864d9012e37f/go.mod h1:RvhuweTFqzezjUFU0SIdTXakrEx9vJlMvQ7znPXSP1g= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go index d2f5471f74..118e8b5dbc 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go @@ -50,13 +50,14 @@ type CapabilitiesData struct { // Capabilities groups several capability aspects type Capabilities struct { - Core *CapabilitiesCore `json:"core" xml:"core"` - Checksums *CapabilitiesChecksums `json:"checksums" xml:"checksums"` - Files *CapabilitiesFiles `json:"files" xml:"files" mapstructure:"files"` - Dav *CapabilitiesDav `json:"dav" xml:"dav"` - FilesSharing *CapabilitiesFilesSharing `json:"files_sharing" xml:"files_sharing" mapstructure:"files_sharing"` - Spaces *Spaces `json:"spaces,omitempty" xml:"spaces,omitempty" mapstructure:"spaces"` - Graph *CapabilitiesGraph `json:"graph,omitempty" xml:"graph,omitempty" mapstructure:"graph"` + Core *CapabilitiesCore `json:"core" xml:"core"` + Checksums *CapabilitiesChecksums `json:"checksums" xml:"checksums"` + Files *CapabilitiesFiles `json:"files" xml:"files" mapstructure:"files"` + Dav *CapabilitiesDav `json:"dav" xml:"dav"` + FilesSharing *CapabilitiesFilesSharing `json:"files_sharing" xml:"files_sharing" mapstructure:"files_sharing"` + Spaces *Spaces `json:"spaces,omitempty" xml:"spaces,omitempty" mapstructure:"spaces"` + Graph *CapabilitiesGraph `json:"graph,omitempty" xml:"graph,omitempty" mapstructure:"graph"` + PasswordPolicies *CapabilitiesPasswordPolicies `json:"password_policies,omitempty" xml:"password_policies,omitempty" mapstructure:"password_policies"` Notifications *CapabilitiesNotifications `json:"notifications,omitempty" xml:"notifications,omitempty"` } @@ -85,6 +86,17 @@ type CapabilitiesGraph struct { Users CapabilitiesGraphUsers `json:"users" xml:"users" mapstructure:"users"` } +// CapabilitiesPasswordPolicies hold the password policies capabilities +type CapabilitiesPasswordPolicies struct { + MinCharacters int `json:"min_characters" xml:"min_characters" mapstructure:"min_characters"` + MaxCharacters int `json:"max_characters" xml:"max_characters" mapstructure:"max_characters"` + MinLowerCaseCharacters int `json:"min_lower_case_characters" xml:"min_lower_case_characters" mapstructure:"min_lower_case_characters"` + MinUpperCaseCharacters int `json:"min_upper_case_characters" xml:"min_upper_case_characters" mapstructure:"min_upper_case_characters"` + MinDigits int `json:"min_digits" xml:"min_digits" mapstructure:"min_digits"` + MinSpecialCharacters int `json:"min_special_characters" xml:"min_special_characters" mapstructure:"min_special_characters"` + SpecialCharacters string `json:"special_characters" xml:"special_characters" mapstructure:"special_characters"` +} + // CapabilitiesGraphUsers holds the graph user capabilities type CapabilitiesGraphUsers struct { ReadOnlyAttributes []string `json:"read_only_attributes" xml:"read_only_attributes" mapstructure:"read_only_attributes"` diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go index 907c233ead..bdb96e3d90 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go @@ -23,7 +23,6 @@ import ( "fmt" "net/http" "strconv" - "strings" permissionsv1beta1 "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -142,7 +141,7 @@ func (h *Handler) createPublicLinkShare(w http.ResponseWriter, r *http.Request, } } - password := strings.TrimSpace(r.FormValue("password")) + password := r.FormValue("password") if h.enforcePassword(permKey) && len(password) == 0 { return nil, &ocsError{ Code: response.MetaBadRequest.StatusCode, @@ -150,6 +149,15 @@ func (h *Handler) createPublicLinkShare(w http.ResponseWriter, r *http.Request, Error: errors.New("missing required password"), } } + if len(password) > 0 { + if err := h.passwordValidator.Validate(password); err != nil { + return nil, &ocsError{ + Code: response.MetaBadRequest.StatusCode, + Message: "password validation failed", + Error: fmt.Errorf("password validation failed: %w", err), + } + } + } if statInfo != nil && statInfo.Type == provider.ResourceType_RESOURCE_TYPE_FILE { // Single file shares should never have delete or create permissions @@ -460,7 +468,7 @@ func (h *Handler) updatePublicShare(w http.ResponseWriter, r *http.Request, shar newPassword, ok := r.Form["password"] // enforcePassword if h.enforcePassword(permKey) { - if (!ok && !share.PasswordProtected) || (ok && len(strings.TrimSpace(newPassword[0])) == 0) { + if !ok && !share.PasswordProtected || ok && len(newPassword[0]) == 0 { response.WriteOCSError(w, r, response.MetaBadRequest.StatusCode, "missing required password", err) return } @@ -468,6 +476,13 @@ func (h *Handler) updatePublicShare(w http.ResponseWriter, r *http.Request, shar // update or clear password if ok { + // skip validation if the clear password scenario + if len(newPassword[0]) > 0 { + if err := h.passwordValidator.Validate(newPassword[0]); err != nil { + response.WriteOCSError(w, r, response.MetaBadRequest.StatusCode, fmt.Errorf("missing required password %w", err).Error(), err) + return + } + } updatesFound = true logger.Info().Str("shares", "update").Msg("password updated") updates = append(updates, &link.UpdatePublicShareRequest_Update{ diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index afe888047b..562187df50 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "fmt" + "log" "mime" "net/http" "path" @@ -39,6 +40,7 @@ import ( link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/v2/pkg/password" "github.com/go-chi/chi/v5" "github.com/rs/zerolog" "google.golang.org/grpc/metadata" @@ -87,6 +89,7 @@ type Handler struct { deniable bool resharing bool publicPasswordEnforced passwordEnforced + passwordValidator password.Validator getClient GatewayClientGetter } @@ -122,7 +125,8 @@ func getCacheWarmupManager(c *config.Config) (sharecache.Warmup, error) { type GatewayClientGetter func() (gateway.GatewayAPIClient, error) // Init initializes this and any contained handlers -func (h *Handler) Init(c *config.Config) { +func (h *Handler) Init(c *config.Config) error { + var err error h.gatewayAddr = c.GatewaySvc h.machineAuthAPIKey = c.MachineAuthAPIKey h.storageRegistryAddr = c.StorageregistrySvc @@ -138,20 +142,29 @@ func (h *Handler) Init(c *config.Config) { h.deniable = c.EnableDenials h.resharing = resharing(c) h.publicPasswordEnforced = publicPwdEnforced(c) + h.passwordValidator, err = passwordPolicies(c) + if err != nil { + return err + } h.statCache = cache.GetStatCache(c.StatCacheStore, c.StatCacheNodes, c.StatCacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize) if c.CacheWarmupDriver != "" { cwm, err := getCacheWarmupManager(c) - if err == nil { - go h.startCacheWarmup(cwm) + if err != nil { + return err } + go h.startCacheWarmup(cwm) } h.getClient = h.getPoolClient + return nil } // InitWithGetter initializes the handler and adds the clientGetter func (h *Handler) InitWithGetter(c *config.Config, clientGetter GatewayClientGetter) { - h.Init(c) + err := h.Init(c) + if err != nil { + log.Fatal(err) + } h.getClient = clientGetter } @@ -1581,6 +1594,30 @@ func publicPwdEnforced(c *config.Config) passwordEnforced { return enf } +func passwordPolicies(c *config.Config) (password.Validator, error) { + var pv password.Validator + var err error + if c.Capabilities.Capabilities == nil || c.Capabilities.Capabilities.PasswordPolicies == nil { + pv, err = password.NewPasswordPolicies(0, 0, 0, 0, 0, "") + if err != nil { + return nil, fmt.Errorf("can't init the Password Policies %w", err) + } + return pv, nil + } + pv, err = password.NewPasswordPolicies( + c.Capabilities.Capabilities.PasswordPolicies.MinCharacters, + c.Capabilities.Capabilities.PasswordPolicies.MinLowerCaseCharacters, + c.Capabilities.Capabilities.PasswordPolicies.MinUpperCaseCharacters, + c.Capabilities.Capabilities.PasswordPolicies.MinDigits, + c.Capabilities.Capabilities.PasswordPolicies.MinSpecialCharacters, + c.Capabilities.Capabilities.PasswordPolicies.SpecialCharacters, + ) + if err != nil { + return nil, fmt.Errorf("can't init the Password Policies %w", err) + } + return pv, nil +} + // sufficientPermissions returns true if the `existing` permissions contain the `requested` permissions func sufficientPermissions(existing, requested *provider.ResourcePermissions, islink bool) bool { ep := conversions.RoleFromResourcePermissions(existing, islink).OCSPermissions() diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/ocs.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/ocs.go index 152bddca4a..09fcfa02d2 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/ocs.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/ocs.go @@ -104,7 +104,10 @@ func (s *svc) routerInit(log *zerolog.Logger) error { capabilitiesHandler.Init(s.c) usersHandler.Init(s.c) configHandler.Init(s.c) - sharesHandler.Init(s.c) + err := sharesHandler.Init(s.c) + if err != nil { + log.Fatal().Msg(err.Error()) + } shareesHandler.Init(s.c) s.router.Route("/v{version:(1|2)}.php", func(r chi.Router) { diff --git a/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go b/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go new file mode 100644 index 0000000000..9ebba7b783 --- /dev/null +++ b/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go @@ -0,0 +1,157 @@ +package password + +import ( + "errors" + "fmt" + "regexp" + "strings" + "unicode/utf8" +) + +// Validator describes the interface providing a password Validate method +type Validator interface { + Validate(str string) error +} + +// Policies represents a password validation rules +type Policies struct { + minCharacters int + minLowerCaseCharacters int + minUpperCaseCharacters int + minDigits int + minSpecialCharacters int + specialCharacters string + digitsRegexp *regexp.Regexp + specialCharactersRegexp *regexp.Regexp +} + +// NewPasswordPolicies returns a new NewPasswordPolicies instance +func NewPasswordPolicies(minCharacters, minLowerCaseCharacters, minUpperCaseCharacters, minDigits, minSpecialCharacters int, + specialCharacters string) (Validator, error) { + p := &Policies{ + minCharacters: minCharacters, + minLowerCaseCharacters: minLowerCaseCharacters, + minUpperCaseCharacters: minUpperCaseCharacters, + minDigits: minDigits, + minSpecialCharacters: minSpecialCharacters, + specialCharacters: specialCharacters, + } + + p.digitsRegexp = regexp.MustCompile("[0-9]") + if len(specialCharacters) > 0 { + var err error + p.specialCharactersRegexp, err = regexp.Compile(specialCharactersExp(specialCharacters)) + if err != nil { + return nil, err + } + } + return p, nil +} + +// Validate implements a password validation regarding the policy +func (s Policies) Validate(str string) error { + var allErr error + if !utf8.ValidString(str) { + return fmt.Errorf("the password contains invalid characters") + } + err := s.validateCharacters(str) + if err != nil { + allErr = errors.Join(allErr, err) + } + err = s.validateLowerCase(str) + if err != nil { + allErr = errors.Join(allErr, err) + } + err = s.validateUpperCase(str) + if err != nil { + allErr = errors.Join(allErr, err) + } + err = s.validateDigits(str) + if err != nil { + allErr = errors.Join(allErr, err) + } + err = s.validateSpecialCharacters(str) + if err != nil { + allErr = errors.Join(allErr, err) + } + if allErr != nil { + return allErr + } + return nil +} + +func (s Policies) validateCharacters(str string) error { + if s.count(str) < s.minCharacters { + return fmt.Errorf("at least %d characters are required", s.minCharacters) + } + return nil +} + +func (s Policies) validateLowerCase(str string) error { + if s.countLowerCaseCharacters(str) < s.minLowerCaseCharacters { + return fmt.Errorf("at least %d lowercase letters are required", s.minLowerCaseCharacters) + } + return nil +} + +func (s Policies) validateUpperCase(str string) error { + if s.countUpperCaseCharacters(str) < s.minUpperCaseCharacters { + return fmt.Errorf("at least %d uppercase letters are required", s.minUpperCaseCharacters) + } + return nil +} + +func (s Policies) validateDigits(str string) error { + if s.countDigits(str) < s.minDigits { + return fmt.Errorf("at least %d numbers are required", s.minDigits) + } + return nil +} + +func (s Policies) validateSpecialCharacters(str string) error { + if s.countSpecialCharacters(str) < s.minSpecialCharacters { + return fmt.Errorf("at least %d special characters are required. %s", s.minSpecialCharacters, s.specialCharacters) + } + return nil +} + +func (s Policies) count(str string) int { + return utf8.RuneCount([]byte(str)) +} + +func (s Policies) countLowerCaseCharacters(str string) int { + var count int + for _, c := range str { + if strings.ToLower(string(c)) == string(c) && strings.ToUpper(string(c)) != string(c) { + count++ + } + } + return count +} + +func (s Policies) countUpperCaseCharacters(str string) int { + var count int + for _, c := range str { + if strings.ToUpper(string(c)) == string(c) && strings.ToLower(string(c)) != string(c) { + count++ + } + } + return count +} + +func (s Policies) countDigits(str string) int { + return len(s.digitsRegexp.FindAllStringIndex(str, -1)) +} + +func (s Policies) countSpecialCharacters(str string) int { + if s.specialCharactersRegexp == nil { + return 0 + } + res := s.specialCharactersRegexp.FindAllStringIndex(str, -1) + return len(res) +} + +func specialCharactersExp(str string) string { + // escape the '-' character because it is a not meta-characters but, they are special inside of [] + return "[" + strings.ReplaceAll(regexp.QuoteMeta(str), "-", `\-`) + "]" +} diff --git a/vendor/github.com/cs3org/reva/v2/pkg/utils/grpc.go b/vendor/github.com/cs3org/reva/v2/pkg/utils/grpc.go index 90547443fa..662a0b32f9 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/utils/grpc.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/utils/grpc.go @@ -2,27 +2,30 @@ package utils import ( "context" + "encoding/json" + "errors" "fmt" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" revactx "github.com/cs3org/reva/v2/pkg/ctx" + "google.golang.org/grpc/metadata" ) -// GetUser gets the specified user -func GetUser(userID *user.UserId, gwc gateway.GatewayAPIClient) (*user.User, error) { - getUserResponse, err := gwc.GetUser(context.Background(), &user.GetUserRequest{UserId: userID}) - if err != nil { - return nil, err - } - if getUserResponse.Status.Code != rpc.Code_CODE_OK { - return nil, fmt.Errorf("error getting user: %s", getUserResponse.Status.Message) - } +// SpaceRole defines the user role on space +type SpaceRole func(*storageprovider.ResourcePermissions) bool - return getUserResponse.GetUser(), nil -} +// Possible roles in spaces +var ( + AllRole SpaceRole = func(perms *storageprovider.ResourcePermissions) bool { return true } + ViewerRole SpaceRole = func(perms *storageprovider.ResourcePermissions) bool { return perms.Stat } + EditorRole SpaceRole = func(perms *storageprovider.ResourcePermissions) bool { return perms.InitiateFileUpload } + ManagerRole SpaceRole = func(perms *storageprovider.ResourcePermissions) bool { return perms.DenyGrant } +) // GetServiceUserContext returns an authenticated context of the given service user func GetServiceUserContext(serviceUserID string, gwc gateway.GatewayAPIClient, serviceUserSecret string) (context.Context, error) { @@ -41,3 +44,174 @@ func GetServiceUserContext(serviceUserID string, gwc gateway.GatewayAPIClient, s return metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, authRes.Token), nil } + +// GetUser gets the specified user +func GetUser(userID *user.UserId, gwc gateway.GatewayAPIClient) (*user.User, error) { + getUserResponse, err := gwc.GetUser(context.Background(), &user.GetUserRequest{UserId: userID}) + if err != nil { + return nil, err + } + if getUserResponse.Status.Code != rpc.Code_CODE_OK { + return nil, fmt.Errorf("error getting user: %s", getUserResponse.Status.Message) + } + + return getUserResponse.GetUser(), nil +} + +// GetSpace returns the given space +func GetSpace(ctx context.Context, spaceID string, gwc gateway.GatewayAPIClient) (*storageprovider.StorageSpace, error) { + res, err := gwc.ListStorageSpaces(ctx, listStorageSpaceRequest(spaceID)) + if err != nil { + return nil, err + } + + if res.GetStatus().GetCode() != rpc.Code_CODE_OK { + return nil, fmt.Errorf("error while getting space: (%v) %s", res.GetStatus().GetCode(), res.GetStatus().GetMessage()) + } + + if len(res.StorageSpaces) == 0 { + return nil, fmt.Errorf("error getting storage space %s: no space returned", spaceID) + } + + return res.StorageSpaces[0], nil +} + +// GetGroupMembers returns all members of the given group +func GetGroupMembers(ctx context.Context, groupID string, gwc gateway.GatewayAPIClient) ([]string, error) { + r, err := gwc.GetGroup(ctx, &group.GetGroupRequest{GroupId: &group.GroupId{OpaqueId: groupID}}) + if err != nil { + return nil, err + } + + if r.GetStatus().GetCode() != rpc.Code_CODE_OK { + return nil, fmt.Errorf("unexpected status code from gateway client: %d", r.GetStatus().GetCode()) + } + + users := make([]string, 0, len(r.GetGroup().GetMembers())) + for _, u := range r.GetGroup().GetMembers() { + users = append(users, u.GetOpaqueId()) + } + + return users, nil +} + +// ResolveID returns either the given userID or all members of the given groupID (if userID is nil) +func ResolveID(ctx context.Context, userid *user.UserId, groupid *group.GroupId, gwc gateway.GatewayAPIClient) ([]string, error) { + if userid != nil { + return []string{userid.GetOpaqueId()}, nil + } + + if ctx == nil { + return nil, errors.New("need ctx to resolve group id") + } + + return GetGroupMembers(ctx, groupid.GetOpaqueId(), gwc) +} + +// GetSpaceMembers returns all members of the given space that have at least the given role. `nil` role will be interpreted as all +func GetSpaceMembers(ctx context.Context, spaceID string, gwc gateway.GatewayAPIClient, role SpaceRole) ([]string, error) { + if ctx == nil { + return nil, errors.New("need authenticated context to find space members") + } + + space, err := GetSpace(ctx, spaceID, gwc) + if err != nil { + return nil, err + } + + var users []string + switch space.SpaceType { + case "personal": + users = append(users, space.GetOwner().GetId().GetOpaqueId()) + case "project": + if users, err = gatherProjectSpaceMembers(ctx, space, gwc, role); err != nil { + return nil, err + } + default: + // TODO: shares? other space types? + return nil, fmt.Errorf("unsupported space type: %s", space.SpaceType) + } + + return users, nil +} + +// GetResourceByID is a convenience method to get a resource by its resourceID +func GetResourceByID(ctx context.Context, resourceid *storageprovider.ResourceId, gwc gateway.GatewayAPIClient) (*storageprovider.ResourceInfo, error) { + return GetResource(ctx, &storageprovider.Reference{ResourceId: resourceid}, gwc) +} + +// GetResource returns a resource by reference +func GetResource(ctx context.Context, ref *storageprovider.Reference, gwc gateway.GatewayAPIClient) (*storageprovider.ResourceInfo, error) { + res, err := gwc.Stat(ctx, &storageprovider.StatRequest{Ref: ref}) + if err != nil { + return nil, err + } + + if res.GetStatus().GetCode() != rpc.Code_CODE_OK { + return nil, fmt.Errorf("unexpected status code while getting space: %v", res.GetStatus().GetCode()) + } + + return res.GetInfo(), nil +} + +func gatherProjectSpaceMembers(ctx context.Context, space *storageprovider.StorageSpace, gwc gateway.GatewayAPIClient, role SpaceRole) ([]string, error) { + var permissionsMap map[string]*storageprovider.ResourcePermissions + if err := ReadJSONFromOpaque(space.GetOpaque(), "grants", &permissionsMap); err != nil { + return nil, err + } + + groupsMap := make(map[string]struct{}) + if opaqueGroups, ok := space.Opaque.Map["groups"]; ok { + _ = json.Unmarshal(opaqueGroups.GetValue(), &groupsMap) + } + + if role == nil { + role = AllRole + } + + // we use a map to avoid duplicates + usermap := make(map[string]struct{}) + for id, perm := range permissionsMap { + if !role(perm) { + continue + } + + if _, isGroup := groupsMap[id]; !isGroup { + usermap[id] = struct{}{} + continue + } + + usrs, err := GetGroupMembers(ctx, id, gwc) + if err != nil { + // TODO: continue? + return nil, err + } + + for _, u := range usrs { + usermap[u] = struct{}{} + } + } + + users := make([]string, 0, len(usermap)) + for id := range usermap { + users = append(users, id) + } + + return users, nil +} + +func listStorageSpaceRequest(spaceID string) *storageprovider.ListStorageSpacesRequest { + return &storageprovider.ListStorageSpacesRequest{ + Opaque: AppendPlainToOpaque(nil, "unrestricted", "true"), + Filters: []*storageprovider.ListStorageSpacesRequest_Filter{ + { + Type: storageprovider.ListStorageSpacesRequest_Filter_TYPE_ID, + Term: &storageprovider.ListStorageSpacesRequest_Filter_Id{ + Id: &storageprovider.StorageSpaceId{ + OpaqueId: spaceID, + }, + }, + }, + }, + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 20f7173fe1..8f3af3a0f5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -356,7 +356,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.16.1-0.20230904124812-2ebd3e92cdb9 +# github.com/cs3org/reva/v2 v2.16.1-0.20230906142214-864d9012e37f ## explicit; go 1.20 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime @@ -554,6 +554,7 @@ github.com/cs3org/reva/v2/pkg/ocm/share/manager/loader github.com/cs3org/reva/v2/pkg/ocm/share/manager/nextcloud github.com/cs3org/reva/v2/pkg/ocm/share/manager/registry github.com/cs3org/reva/v2/pkg/ocm/share/sender +github.com/cs3org/reva/v2/pkg/password github.com/cs3org/reva/v2/pkg/permission github.com/cs3org/reva/v2/pkg/permission/manager/demo github.com/cs3org/reva/v2/pkg/permission/manager/loader From 54e5eb03f93f12a24f8a64c27d851c97880c5534 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 7 Sep 2023 13:57:00 +0200 Subject: [PATCH 6/6] remove commented code and add portrange Signed-off-by: jkoberg --- docs/services/general-info/port-ranges.md | 6 ++--- services/clientlog/pkg/command/server.go | 9 ------- .../pkg/service/service_suit_test.go | 26 ------------------- .../clientlog/pkg/service/service_test.go | 3 --- 4 files changed, 3 insertions(+), 41 deletions(-) delete mode 100644 services/clientlog/pkg/service/service_suit_test.go delete mode 100644 services/clientlog/pkg/service/service_test.go diff --git a/docs/services/general-info/port-ranges.md b/docs/services/general-info/port-ranges.md index e3baac04d5..2252255da4 100644 --- a/docs/services/general-info/port-ranges.md +++ b/docs/services/general-info/port-ranges.md @@ -52,7 +52,7 @@ We also suggest to use the last port in your extensions' range as a debug/metric | 9195-9199 | FREE | | 9200-9204 | [proxy]({{< ref "../proxy/_index.md" >}}) | | 9205-9209 | [proxy]({{< ref "../proxy/_index.md" >}}) | -| 9210-9214 | [userlog]{{< ref "../userlog/_index.md" >}} | +| 9210-9214 | [userlog]({{< ref "../userlog/_index.md" >}}) | | 9215-9219 | [storage-system]({{< ref "../storage-system/_index.md" >}}) | | 9220-9224 | [search]({{< ref "../search/_index.md" >}}) | | 9225-9229 | [audit]({{< ref "../audit/_index.md" >}}) | @@ -61,8 +61,8 @@ We also suggest to use the last port in your extensions' range as a debug/metric | 9240-9244 | [app-registry]({{< ref "../app-registry/_index.md" >}}) | | 9245-9249 | FREE | | 9250-9254 | [ocis server (runtime)](https://github.com/owncloud/ocis/tree/master/ocis/pkg/runtime) | -| 9255-9259 | [postprocessing]({{ ref "../postprocessing/_index.md" >}}) | -| 9260-9264 | FREE | +| 9255-9259 | [postprocessing]({{< ref "../postprocessing/_index.md" >}}) | +| 9260-9264 | [clientlog]({{ ref "../clientlog/index.md" }}) | | 9265-9269 | FREE | | 9270-9274 | [eventhistory]({{< ref "../eventhistory/_index.md" >}}) | | 9275-9279 | FREE | diff --git a/services/clientlog/pkg/command/server.go b/services/clientlog/pkg/command/server.go index b149a6ce27..050849c0b8 100644 --- a/services/clientlog/pkg/command/server.go +++ b/services/clientlog/pkg/command/server.go @@ -43,15 +43,6 @@ func Server(cfg *config.Config) *cli.Command { return err } - /* - grpcClient, err := ogrpc.NewClient( - append(ogrpc.GetClientOptions(cfg.GRPCClientTLS), ogrpc.WithTraceProvider(tracerProvider))..., - ) - if err != nil { - return err - } - */ // TODO: remove - gr := run.Group{} ctx, cancel := func() (context.Context, context.CancelFunc) { if cfg.Context == nil { diff --git a/services/clientlog/pkg/service/service_suit_test.go b/services/clientlog/pkg/service/service_suit_test.go deleted file mode 100644 index ab7fb7b198..0000000000 --- a/services/clientlog/pkg/service/service_suit_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package service_test - -import ( - "testing" - - "github.com/owncloud/ocis/v2/ocis-pkg/registry" - mRegistry "go-micro.dev/v4/registry" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func init() { - registry.Configure("memory") - r := registry.GetRegistry() - service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") - service.Nodes = []*mRegistry.Node{{ - Address: "any", - }} - - _ = r.Register(service) -} -func TestSearch(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Userlog service Suite") -} diff --git a/services/clientlog/pkg/service/service_test.go b/services/clientlog/pkg/service/service_test.go deleted file mode 100644 index 922fe8e45c..0000000000 --- a/services/clientlog/pkg/service/service_test.go +++ /dev/null @@ -1,3 +0,0 @@ -package service_test - -// TODO: TEST!