[full-ci] Events (#3146)

* initial events draft

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* start nats server with ocis

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* use feature reva and add config

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* remove unneeded files

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* edge reva and fix pipeline

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* configure nats server via ocis

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* update expected failures

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* add nats to parallel deployment

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* more expected failures

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* use NATS instead of cp SETTINGS

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* remove more configuration options

Signed-off-by: jkoberg <jkoberg@owncloud.com>

.drone.star

@@ -1491,7 +1491,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "OCIS_STORAGE_READ_ONLY": "false",
             # General oCIS config
             # OCIS_RUN_EXTENSIONS specifies to start all extensions except glauth, idp and accounts. These are replaced by external services
-            "OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy",
+            "OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,nats",
             "OCIS_LOG_LEVEL": "error",
             "OCIS_URL": OCIS_URL,
             "PROXY_TLS": "true",

deployments/examples/oc10_ocis_parallel/docker-compose.yml

@@ -112,7 +112,7 @@ services:
       OCIS_STORAGE_READ_ONLY: "false" # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303
       # General oCIS config
       # OCIS_RUN_EXTENSIONS specifies to start all extensions except glauth, idp and accounts. These are replaced by external services
-      OCIS_RUN_EXTENSIONS: settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy
+      OCIS_RUN_EXTENSIONS: settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,nats
       OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose
       OCIS_URL: https://${CLOUD_DOMAIN:-cloud.owncloud.test}
       PROXY_TLS: "false" # do not use SSL between Traefik and oCIS

go.mod

@@ -44,6 +44,7 @@ require (
 	github.com/mennanov/fieldmask-utils v0.5.0
 	github.com/mitchellh/mapstructure v1.4.3
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
+	github.com/nats-io/nats-streaming-server v0.24.1
 	github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484
 	github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba
 	github.com/oklog/run v1.1.0
@@ -200,7 +201,6 @@ require (
 	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 // indirect
 	github.com/nats-io/nats-server/v2 v2.7.2 // indirect
-	github.com/nats-io/nats-streaming-server v0.24.1 // indirect
 	github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect

nats/cmd/nats/main.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"os"
+
+	"github.com/owncloud/ocis/nats/pkg/command"
+	"github.com/owncloud/ocis/nats/pkg/config"
+)
+
+func main() {
+	if err := command.Execute(config.DefaultConfig()); err != nil {
+		os.Exit(1)
+	}
+}

nats/pkg/command/health.go

@@ -0,0 +1,18 @@
+package command
+
+import (
+	"github.com/owncloud/ocis/nats/pkg/config"
+	"github.com/urfave/cli/v2"
+)
+
+// Health is the entrypoint for the health command.
+func Health(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "health",
+		Usage: "Check health status",
+		Action: func(c *cli.Context) error {
+			// Not implemented
+			return nil
+		},
+	}
+}

nats/pkg/command/root.go

@@ -0,0 +1,64 @@
+package command
+
+import (
+	"context"
+	"os"
+
+	"github.com/owncloud/ocis/nats/pkg/config"
+	"github.com/owncloud/ocis/ocis-pkg/clihelper"
+	ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
+	"github.com/thejerf/suture/v4"
+	"github.com/urfave/cli/v2"
+)
+
+// GetCommands provides all commands for this service
+func GetCommands(cfg *config.Config) cli.Commands {
+	return []*cli.Command{
+		// start this service
+		Server(cfg),
+
+		// interaction with this service
+
+		// infos about this service
+		Health(cfg),
+		Version(cfg),
+	}
+}
+
+// Execute is the entry point for the nats command.
+func Execute(cfg *config.Config) error {
+	app := clihelper.DefaultApp(&cli.App{
+		Name:     "nats",
+		Usage:    "starts nats server",
+		Commands: GetCommands(cfg),
+	})
+
+	cli.HelpFlag = &cli.BoolFlag{
+		Name:  "help,h",
+		Usage: "Show the help",
+	}
+
+	return app.Run(os.Args)
+}
+
+// SutureService allows for the nats command to be embedded and supervised by a suture supervisor tree.
+type SutureService struct {
+	cfg *config.Config
+}
+
+// NewSutureService creates a new nats.SutureService
+func NewSutureService(cfg *ociscfg.Config) suture.Service {
+	cfg.Settings.Commons = cfg.Commons
+	return SutureService{
+		cfg: cfg.Nats,
+	}
+}
+
+func (s SutureService) Serve(ctx context.Context) error {
+	s.cfg.Context = ctx
+	if err := Execute(s.cfg); err != nil {
+		return err
+	}
+
+	return nil
+}

nats/pkg/command/server.go

@@ -0,0 +1,79 @@
+package command
+
+import (
+	"fmt"
+
+	"github.com/cs3org/reva/pkg/events/server"
+	"github.com/owncloud/ocis/nats/pkg/config"
+	"github.com/owncloud/ocis/nats/pkg/config/parser"
+	"github.com/owncloud/ocis/nats/pkg/logging"
+	"github.com/owncloud/ocis/ocis-pkg/log"
+	"github.com/urfave/cli/v2"
+
+	// TODO: .Logger Option on events/server would make this import redundant
+	stanServer "github.com/nats-io/nats-streaming-server/server"
+)
+
+// Server is the entrypoint for the server command.
+func Server(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:     "server",
+		Usage:    fmt.Sprintf("start %s extension without runtime (unsupervised mode)", cfg.Service.Name),
+		Category: "server",
+		Before: func(c *cli.Context) error {
+			return parser.ParseConfig(cfg)
+		},
+		Action: func(c *cli.Context) error {
+			logger := logging.Configure(cfg.Service.Name, cfg.Log)
+			err := server.RunNatsServer(server.Host(cfg.Nats.Host), server.Port(cfg.Nats.Port), server.StanOpts(func(o *stanServer.Options) {
+				o.CustomLogger = &logWrapper{logger}
+			}))
+			if err != nil {
+				return err
+			}
+			for {
+			}
+		},
+	}
+}
+
+// we need to wrap our logger so we can pass it to the nats server
+type logWrapper struct {
+	logger log.Logger
+}
+
+// Noticef logs a notice statement
+func (l *logWrapper) Noticef(format string, v ...interface{}) {
+	msg := fmt.Sprintf(format, v...)
+	l.logger.Info().Msg(msg)
+}
+
+// Warnf logs a warning statement
+func (l *logWrapper) Warnf(format string, v ...interface{}) {
+	msg := fmt.Sprintf(format, v...)
+	l.logger.Warn().Msg(msg)
+}
+
+// Fatalf logs a fatal statement
+func (l *logWrapper) Fatalf(format string, v ...interface{}) {
+	msg := fmt.Sprintf(format, v...)
+	l.logger.Fatal().Msg(msg)
+}
+
+// Errorf logs an error statement
+func (l *logWrapper) Errorf(format string, v ...interface{}) {
+	msg := fmt.Sprintf(format, v...)
+	l.logger.Error().Msg(msg)
+}
+
+// Debugf logs a debug statement
+func (l *logWrapper) Debugf(format string, v ...interface{}) {
+	msg := fmt.Sprintf(format, v...)
+	l.logger.Debug().Msg(msg)
+}
+
+// Tracef logs a trace statement
+func (l *logWrapper) Tracef(format string, v ...interface{}) {
+	msg := fmt.Sprintf(format, v...)
+	l.logger.Trace().Msg(msg)
+}

nats/pkg/command/version.go

@@ -0,0 +1,19 @@
+package command
+
+import (
+	"github.com/owncloud/ocis/nats/pkg/config"
+	"github.com/urfave/cli/v2"
+)
+
+// Version prints the service versions of all running instances.
+func Version(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:     "version",
+		Usage:    "print the version of this binary and the running extension instances",
+		Category: "info",
+		Action: func(c *cli.Context) error {
+			// not implemented
+			return nil
+		},
+	}
+}

nats/pkg/config/config.go

@@ -0,0 +1,27 @@
+package config
+
+import (
+	"context"
+
+	"github.com/owncloud/ocis/ocis-pkg/shared"
+)
+
+// Config combines all available configuration parts.
+type Config struct {
+	*shared.Commons
+
+	Service Service
+
+	Log   *Log  `ocisConfig:"log"`
+	Debug Debug `ocisConfig:"debug"`
+
+	Nats Nats `ociConfig:"nats"`
+
+	Context context.Context
+}
+
+// Nats is the nats config
+type Nats struct {
+	Host string
+	Port int
+}

nats/pkg/config/debug.go

@@ -0,0 +1,9 @@
+package config
+
+// Debug defines the available debug configuration.
+type Debug struct {
+	Addr   string `ocisConfig:"addr" env:"NATS_DEBUG_ADDR"`
+	Token  string `ocisConfig:"token" env:"NATS_DEBUG_TOKEN"`
+	Pprof  bool   `ocisConfig:"pprof" env:"NATS_DEBUG_PPROF"`
+	Zpages bool   `ocisConfig:"zpages" env:"NATS_DEBUG_ZPAGES"`
+}

nats/pkg/config/defaultconfig.go

@@ -0,0 +1,16 @@
+package config
+
+// NOTE: Most of this configuration is not needed to keep it as simple as possible
+// TODO: Clean up unneeded configuration
+
+func DefaultConfig() *Config {
+	return &Config{
+		Service: Service{
+			Name: "nats",
+		},
+		Nats: Nats{
+			Host: "127.0.0.1",
+			Port: 4222,
+		},
+	}
+}

nats/pkg/config/log.go

@@ -0,0 +1,9 @@
+package config
+
+// Log defines the available log configuration.
+type Log struct {
+	Level  string `mapstructure:"level" env:"OCIS_LOG_LEVEL;NATS_LOG_LEVEL"`
+	Pretty bool   `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;NATS_LOG_PRETTY"`
+	Color  bool   `mapstructure:"color" env:"OCIS_LOG_COLOR;NATS_LOG_COLOR"`
+	File   string `mapstructure:"file" env:"OCIS_LOG_FILE;NATS_LOG_FILE"`
+}

nats/pkg/config/parser/parse.go

@@ -0,0 +1,40 @@
+package parser
+
+import (
+	"errors"
+
+	"github.com/owncloud/ocis/nats/pkg/config"
+	ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
+
+	"github.com/owncloud/ocis/ocis-pkg/config/envdecode"
+)
+
+// ParseConfig loads accounts configuration from known paths.
+func ParseConfig(cfg *config.Config) error {
+	_, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg)
+	if err != nil {
+		return err
+	}
+
+	// provide with defaults for shared logging, since we need a valid destination address for BindEnv.
+	if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
+		cfg.Log = &config.Log{
+			Level:  cfg.Commons.Log.Level,
+			Pretty: cfg.Commons.Log.Pretty,
+			Color:  cfg.Commons.Log.Color,
+			File:   cfg.Commons.Log.File,
+		}
+	} else if cfg.Log == nil {
+		cfg.Log = &config.Log{}
+	}
+
+	// load all env variables relevant to the config in the current context.
+	if err := envdecode.Decode(cfg); err != nil {
+		// no environment variable set for this config is an expected "error"
+		if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
+			return err
+		}
+	}
+
+	return nil
+}

nats/pkg/config/service.go

@@ -0,0 +1,6 @@
+package config
+
+// Service defines the available service configuration.
+type Service struct {
+	Name string
+}

nats/pkg/logging/logging.go

@@ -0,0 +1,17 @@
+package logging
+
+import (
+	"github.com/owncloud/ocis/nats/pkg/config"
+	"github.com/owncloud/ocis/ocis-pkg/log"
+)
+
+// LoggerFromConfig initializes a service-specific logger instance.
+func Configure(name string, cfg *config.Log) log.Logger {
+	return log.NewLogger(
+		log.Name(name),
+		log.Level(cfg.Level),
+		log.Pretty(cfg.Pretty),
+		log.Color(cfg.Color),
+		log.File(cfg.File),
+	)
+}

ocis-pkg/config/config.go

@@ -8,6 +8,7 @@ import (
 	graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/config"
 	graph "github.com/owncloud/ocis/graph/pkg/config"
 	idp "github.com/owncloud/ocis/idp/pkg/config"
+	nats "github.com/owncloud/ocis/nats/pkg/config"
 	ocs "github.com/owncloud/ocis/ocs/pkg/config"
 	proxy "github.com/owncloud/ocis/proxy/pkg/config"
 	settings "github.com/owncloud/ocis/settings/pkg/config"
@@ -60,6 +61,7 @@ type Config struct {
 	Graph         *graph.Config         `ocisConfig:"graph"`
 	GraphExplorer *graphExplorer.Config `ocisConfig:"graph_explorer"`
 	IDP           *idp.Config           `ocisConfig:"idp"`
+	Nats          *nats.Config          `ocisConfig:"nats"`
 	OCS           *ocs.Config           `ocisConfig:"ocs"`
 	Web           *web.Config           `ocisConfig:"web"`
 	Proxy         *proxy.Config         `ocisConfig:"proxy"`

ocis-pkg/config/defaultconfig.go

@@ -6,6 +6,7 @@ import (
 	graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/config"
 	graph "github.com/owncloud/ocis/graph/pkg/config"
 	idp "github.com/owncloud/ocis/idp/pkg/config"
+	nats "github.com/owncloud/ocis/nats/pkg/config"
 	ocs "github.com/owncloud/ocis/ocs/pkg/config"
 	proxy "github.com/owncloud/ocis/proxy/pkg/config"
 	settings "github.com/owncloud/ocis/settings/pkg/config"
@@ -29,6 +30,7 @@ func DefaultConfig() *Config {
 		GLAuth:        glauth.DefaultConfig(),
 		Graph:         graph.DefaultConfig(),
 		IDP:           idp.DefaultConfig(),
+		Nats:          nats.DefaultConfig(),
 		Proxy:         proxy.DefaultConfig(),
 		GraphExplorer: graphExplorer.DefaultConfig(),
 		OCS:           ocs.DefaultConfig(),

ocis/pkg/command/natsserver.go

@@ -0,0 +1,26 @@
+package command
+
+import (
+	"github.com/owncloud/ocis/nats/pkg/command"
+	"github.com/owncloud/ocis/ocis-pkg/config"
+	"github.com/owncloud/ocis/ocis-pkg/config/parser"
+	"github.com/owncloud/ocis/ocis/pkg/register"
+	"github.com/urfave/cli/v2"
+)
+
+// NatsServerCommand is the entrypoint for the nats server command.
+func NatsServerCommand(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:     "nats-server",
+		Usage:    "start nats server",
+		Category: "extensions",
+		Before: func(ctx *cli.Context) error {
+			return parser.ParseConfig(cfg)
+		},
+		Subcommands: command.GetCommands(cfg.Nats),
+	}
+}
+
+func init() {
+	register.AddCommand(NatsServerCommand)
+}

ocis/pkg/runtime/service/service.go

@@ -23,6 +23,7 @@ import (
 	graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/command"
 	graph "github.com/owncloud/ocis/graph/pkg/command"
 	idp "github.com/owncloud/ocis/idp/pkg/command"
+	nats "github.com/owncloud/ocis/nats/pkg/command"
 	"github.com/owncloud/ocis/ocis-pkg/config"
 	ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
 	"github.com/owncloud/ocis/ocis-pkg/log"
@@ -91,6 +92,7 @@ func NewService(options ...Option) (*Service, error) {
 	}
 
 	s.ServicesRegistry["settings"] = settings.NewSutureService
+	s.ServicesRegistry["nats"] = nats.NewSutureService
 	s.ServicesRegistry["storage-metadata"] = storage.NewStorageMetadata
 	s.ServicesRegistry["glauth"] = glauth.NewSutureService
 	s.ServicesRegistry["graph"] = graph.NewSutureService

storage/pkg/command/sharing.go

@@ -175,6 +175,14 @@ func sharingConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]inte
 					},
 				},
 			},
+			"interceptors": map[string]interface{}{
+				"eventsmiddleware": map[string]interface{}{
+					"group":     "sharing",
+					"type":      "nats",
+					"address":   "127.0.0.1:4222",
+					"clusterID": "test-cluster",
+				},
+			},
 		},
 	}
 	return rcfg