mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-04-24 04:58:31 -05:00
Merge pull request #5600 from kobergj/EventHistory
Eventhistory Service
This commit is contained in:
kobergj
@@ -0,0 +1,37 @@
|
||||
SHELL := bash
|
||||
NAME := eventhistory
|
||||
|
||||
include ../../.make/recursion.mk
|
||||
|
||||
############ tooling ############
|
||||
ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI
|
||||
include ../../.bingo/Variables.mk
|
||||
endif
|
||||
|
||||
############ go tooling ############
|
||||
include ../../.make/go.mk
|
||||
|
||||
############ release ############
|
||||
include ../../.make/release.mk
|
||||
|
||||
############ docs generate ############
|
||||
include ../../.make/docs.mk
|
||||
|
||||
.PHONY: docs-generate
|
||||
docs-generate: config-docs-generate
|
||||
|
||||
############ generate ############
|
||||
include ../../.make/generate.mk
|
||||
|
||||
.PHONY: ci-go-generate
|
||||
ci-go-generate: # CI runs ci-node-generate automatically before this target
|
||||
|
||||
.PHONY: ci-node-generate
|
||||
ci-node-generate:
|
||||
|
||||
############ licenses ############
|
||||
.PHONY: ci-node-check-licenses
|
||||
ci-node-check-licenses:
|
||||
|
||||
.PHONY: ci-node-save-licenses
|
||||
ci-node-save-licenses:
|
||||
@@ -0,0 +1,29 @@
|
||||
# Eventhistory Service
|
||||
|
||||
The `eventhistory` consumes all events from the configured event system like NATS, stores them and allows other services to retrieve them via an eventid.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Running the eventhistory service without an event sytem like NATS is not possible.
|
||||
|
||||
## Consuming
|
||||
|
||||
The `eventhistory` services consumes all events from the configured event sytem.
|
||||
|
||||
## Storing
|
||||
|
||||
The `eventhistory` service stores each consumed event via the configured store in `EVENTHISTORY_STORE_TYPE`. Possible stores are:
|
||||
- `mem`: Basic in-memory store and the default.
|
||||
- `ocmem`: Advanced in-memory store allowing max size.
|
||||
- `redis`: Stores data in a configured redis cluster.
|
||||
- `etcd`: Stores data in a configured etcd cluster.
|
||||
- `nats-js`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in productive enviroments.
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot persistent.
|
||||
2. Though usually not necessary, a database name and a database table can be configured for event stores if the event store supports this. Generally not applicapable for stores of type `in-memory`. These settings are blank by default which means that the standard settings of the configured store applies.
|
||||
3. Events stay in the store for 2 weeks by default. Use `EVENTHISTORY_RECORD_EXPIRY` to adjust this value.
|
||||
|
||||
## Retrieving
|
||||
|
||||
Other services can call the `eventhistory` service via a grpc call to retrieve events. The request must contain the eventid that should be retrieved.
|
||||
@@ -0,0 +1,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/command"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config/defaults"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := command.Execute(defaults.DefaultConfig()); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// Health is the entrypoint for the health command.
|
||||
func Health(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "health",
|
||||
Usage: "Check health status",
|
||||
Action: func(c *cli.Context) error {
|
||||
// Not implemented
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/clihelper"
|
||||
ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
|
||||
"github.com/thejerf/suture/v4"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// GetCommands provides all commands for this service
|
||||
func GetCommands(cfg *config.Config) cli.Commands {
|
||||
return []*cli.Command{
|
||||
// start this service
|
||||
Server(cfg),
|
||||
|
||||
// interaction with this service
|
||||
|
||||
// infos about this service
|
||||
Health(cfg),
|
||||
Version(cfg),
|
||||
}
|
||||
}
|
||||
|
||||
// Execute is the entry point for the eventhistory command.
|
||||
func Execute(cfg *config.Config) error {
|
||||
app := clihelper.DefaultApp(&cli.App{
|
||||
Name: "eventhistory",
|
||||
Usage: "starts eventhistory service",
|
||||
Commands: GetCommands(cfg),
|
||||
})
|
||||
|
||||
return app.Run(os.Args)
|
||||
}
|
||||
|
||||
// SutureService allows for the eventhistory command to be embedded and supervised by a suture supervisor tree.
|
||||
type SutureService struct {
|
||||
cfg *config.Config
|
||||
}
|
||||
|
||||
// NewSutureService creates a new eventhistory.SutureService
|
||||
func NewSutureService(cfg *ociscfg.Config) suture.Service {
|
||||
cfg.Notifications.Commons = cfg.Commons
|
||||
return SutureService{
|
||||
cfg: cfg.EventHistory,
|
||||
}
|
||||
}
|
||||
|
||||
func (s SutureService) Serve(ctx context.Context) error {
|
||||
s.cfg.Context = ctx
|
||||
if err := Execute(s.cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/events/stream"
|
||||
"github.com/oklog/run"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
|
||||
ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/store"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/version"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config/parser"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/logging"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/metrics"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/server/grpc"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// Server is the entrypoint for the server command.
|
||||
func Server(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "server",
|
||||
Usage: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name),
|
||||
Category: "server",
|
||||
Before: func(c *cli.Context) error {
|
||||
return configlog.ReturnFatal(parser.ParseConfig(cfg))
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
logger := logging.Configure(cfg.Service.Name, cfg.Log)
|
||||
err := ogrpc.Configure(ogrpc.GetClientOptions(cfg.GRPCClientTLS)...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var (
|
||||
gr = run.Group{}
|
||||
ctx, cancel = func() (context.Context, context.CancelFunc) {
|
||||
if cfg.Context == nil {
|
||||
return context.WithCancel(context.Background())
|
||||
}
|
||||
return context.WithCancel(cfg.Context)
|
||||
}()
|
||||
metrics = metrics.New()
|
||||
)
|
||||
|
||||
defer cancel()
|
||||
|
||||
metrics.BuildInfo.WithLabelValues(version.GetString()).Set(1)
|
||||
|
||||
consumer, err := stream.NatsFromConfig(stream.NatsConfig(cfg.Events))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
st := store.Create(
|
||||
store.Type(cfg.Store.Type),
|
||||
store.Addresses(strings.Split(cfg.Store.Addresses, ",")...),
|
||||
store.Database(cfg.Store.Database),
|
||||
store.Table(cfg.Store.Table),
|
||||
store.TTL(cfg.Store.RecordExpiry),
|
||||
)
|
||||
|
||||
service := grpc.NewService(
|
||||
grpc.Logger(logger),
|
||||
grpc.Context(ctx),
|
||||
grpc.Config(cfg),
|
||||
grpc.Name(cfg.Service.Name),
|
||||
grpc.Namespace(cfg.GRPC.Namespace),
|
||||
grpc.Address(cfg.GRPC.Addr),
|
||||
grpc.Metrics(metrics),
|
||||
grpc.Consumer(consumer),
|
||||
grpc.Store(st),
|
||||
)
|
||||
|
||||
gr.Add(service.Run, func(err error) {
|
||||
logger.Error().
|
||||
Err(err).
|
||||
Str("server", "grpc").
|
||||
Msg("Shutting Down server")
|
||||
|
||||
cancel()
|
||||
})
|
||||
|
||||
return gr.Run()
|
||||
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// Version prints the service versions of all running instances.
|
||||
func Version(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "version",
|
||||
Usage: "print the version of this binary and the running service instances",
|
||||
Category: "info",
|
||||
Action: func(c *cli.Context) error {
|
||||
// not implemented
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
|
||||
)
|
||||
|
||||
// Config combines all available configuration parts.
|
||||
type Config struct {
|
||||
Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service
|
||||
|
||||
Service Service `yaml:"-"`
|
||||
|
||||
Log *Log `yaml:"log"`
|
||||
Debug Debug `yaml:"debug"`
|
||||
|
||||
GRPC GRPCConfig `yaml:"grpc"`
|
||||
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
|
||||
|
||||
Events Events `yaml:"events"`
|
||||
Store Store `yaml:"store"`
|
||||
|
||||
Context context.Context `yaml:"-"`
|
||||
}
|
||||
|
||||
// GRPCConfig defines the available grpc configuration.
|
||||
type GRPCConfig struct {
|
||||
Addr string `ocisConfig:"addr" env:"EVENTHISTORY_GRPC_ADDR" desc:"The bind address of the GRPC service."`
|
||||
Namespace string `ocisConfig:"-" yaml:"-"`
|
||||
TLS *shared.GRPCServiceTLS `yaml:"tls"`
|
||||
}
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Type string `yaml:"type" env:"EVENTHISTORY_STORE_TYPE" desc:"The type of the eventhistory store. Supported values are: 'mem', 'ocmem', 'etcd', 'redis', 'nats-js', 'noop'. See the text description for details."`
|
||||
Addresses string `yaml:"addresses" env:"EVENTHISTORY_STORE_ADDRESSES" desc:"A comma separated list of addresses to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"EVENTHISTORY_STORE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when 'in-memory' stores are configured."`
|
||||
Table string `yaml:"table" env:"EVENTHISTORY_STORE_TABLE" desc:"(optional) The database table the store should use. This has no effect when 'in-memory' stores are configured."`
|
||||
RecordExpiry time.Duration `yaml:"record_expiry" env:"EVENTHISTORY_RECORD_EXPIRY" desc:"Time to life for events in the store. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '336h' (2 weeks)."`
|
||||
Size int `yaml:"size" env:"EVENTHISTORY_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured. Defaults to 512."`
|
||||
}
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
Endpoint string `yaml:"endpoint" env:"EVENTHISTORY_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture."`
|
||||
Cluster string `yaml:"cluster" env:"EVENTHISTORY_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system."`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OCIS_INSECURE;EVENTHISTORY_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates."`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"EVENTHISTORY_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided NOTIFICATIONS_EVENTS_TLS_INSECURE will be seen as false."`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS;EVENTHISTORY_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services.."`
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package config
|
||||
|
||||
// Debug defines the available debug configuration.
|
||||
type Debug struct {
|
||||
Addr string `yaml:"addr" env:"EVENTHISTORY_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed."`
|
||||
Token string `yaml:"token" env:"EVENTHISTORY_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint."`
|
||||
Pprof bool `yaml:"pprof" env:"EVENTHISTORY_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling."`
|
||||
Zpages bool `yaml:"zpages" env:"EVENTHISTORY_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces."`
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package defaults
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
|
||||
)
|
||||
|
||||
// FullDefaultConfig returns the full default config
|
||||
func FullDefaultConfig() *config.Config {
|
||||
cfg := DefaultConfig()
|
||||
EnsureDefaults(cfg)
|
||||
Sanitize(cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// DefaultConfig return the default configuration
|
||||
func DefaultConfig() *config.Config {
|
||||
return &config.Config{
|
||||
Service: config.Service{
|
||||
Name: "eventhistory",
|
||||
},
|
||||
Events: config.Events{
|
||||
Endpoint: "127.0.0.1:9233",
|
||||
Cluster: "ocis-cluster",
|
||||
EnableTLS: false,
|
||||
},
|
||||
Store: config.Store{
|
||||
Type: "mem",
|
||||
RecordExpiry: 336 * time.Hour,
|
||||
},
|
||||
GRPC: config.GRPCConfig{
|
||||
Addr: "127.0.0.1:0",
|
||||
Namespace: "com.owncloud.api",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureDefaults ensures the config contains default values
|
||||
func EnsureDefaults(cfg *config.Config) {
|
||||
// provide with defaults for shared logging, since we need a valid destination address for "envdecode".
|
||||
if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
|
||||
cfg.Log = &config.Log{
|
||||
Level: cfg.Commons.Log.Level,
|
||||
Pretty: cfg.Commons.Log.Pretty,
|
||||
Color: cfg.Commons.Log.Color,
|
||||
File: cfg.Commons.Log.File,
|
||||
}
|
||||
} else if cfg.Log == nil {
|
||||
cfg.Log = &config.Log{}
|
||||
}
|
||||
|
||||
if cfg.GRPCClientTLS == nil {
|
||||
cfg.GRPCClientTLS = &shared.GRPCClientTLS{}
|
||||
if cfg.Commons != nil && cfg.Commons.GRPCClientTLS != nil {
|
||||
cfg.GRPCClientTLS = cfg.Commons.GRPCClientTLS
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.GRPC.TLS == nil {
|
||||
cfg.GRPC.TLS = &shared.GRPCServiceTLS{}
|
||||
if cfg.Commons != nil && cfg.Commons.GRPCServiceTLS != nil {
|
||||
cfg.GRPC.TLS.Enabled = cfg.Commons.GRPCServiceTLS.Enabled
|
||||
cfg.GRPC.TLS.Cert = cfg.Commons.GRPCServiceTLS.Cert
|
||||
cfg.GRPC.TLS.Key = cfg.Commons.GRPCServiceTLS.Key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sanitize sanitizes the config
|
||||
func Sanitize(cfg *config.Config) {
|
||||
// nothing to sanitize here atm
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package config
|
||||
|
||||
// Log defines the available log configuration.
|
||||
type Log struct {
|
||||
Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;EVENTHISTORY_LOG_LEVEL" desc:"The log level. Valid values are: \"panic\", \"fatal\", \"error\", \"warn\", \"info\", \"debug\", \"trace\"."`
|
||||
Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;EVENTHISTORY_LOG_PRETTY" desc:"Activates pretty log output."`
|
||||
Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;EVENTHISTORY_LOG_COLOR" desc:"Activates colorized log output."`
|
||||
File string `mapstructure:"file" env:"OCIS_LOG_FILE;EVENTHISTORY_LOG_FILE" desc:"The path to the log file. Activates logging to this file if set."`
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config/defaults"
|
||||
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/config/envdecode"
|
||||
)
|
||||
|
||||
// ParseConfig loads configuration from known paths.
|
||||
func ParseConfig(cfg *config.Config) error {
|
||||
_, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defaults.EnsureDefaults(cfg)
|
||||
|
||||
// load all env variables relevant to the config in the current context.
|
||||
if err := envdecode.Decode(cfg); err != nil {
|
||||
// no environment variable set for this config is an expected "error"
|
||||
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
defaults.Sanitize(cfg)
|
||||
|
||||
return Validate(cfg)
|
||||
}
|
||||
|
||||
// Validate validates the config
|
||||
func Validate(cfg *config.Config) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package config
|
||||
|
||||
// Service defines the available service configuration.
|
||||
type Service struct {
|
||||
Name string `yaml:"-"`
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
|
||||
)
|
||||
|
||||
// LoggerFromConfig initializes a service-specific logger instance.
|
||||
func Configure(name string, cfg *config.Log) log.Logger {
|
||||
return log.NewLogger(
|
||||
log.Name(name),
|
||||
log.Level(cfg.Level),
|
||||
log.Pretty(cfg.Pretty),
|
||||
log.Color(cfg.Color),
|
||||
log.File(cfg.File),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package metrics
|
||||
|
||||
import "github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
var (
|
||||
// Namespace defines the namespace for the defines metrics.
|
||||
Namespace = "ocis"
|
||||
|
||||
// Subsystem defines the subsystem for the defines metrics.
|
||||
Subsystem = "eventhistory"
|
||||
)
|
||||
|
||||
// Metrics defines the available metrics of this service.
|
||||
type Metrics struct {
|
||||
BuildInfo *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
// New initializes the available metrics.
|
||||
func New() *Metrics {
|
||||
m := &Metrics{
|
||||
BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "build_info",
|
||||
Help: "Build information",
|
||||
}, []string{"version"}),
|
||||
}
|
||||
|
||||
_ = prometheus.Register(
|
||||
m.BuildInfo,
|
||||
)
|
||||
|
||||
// TODO: implement metrics
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/events"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/metrics"
|
||||
"github.com/urfave/cli/v2"
|
||||
"go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Name string
|
||||
Address string
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Metrics *metrics.Metrics
|
||||
Namespace string
|
||||
Flags []cli.Flag
|
||||
Store store.Store
|
||||
Consumer events.Consumer
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// Name provides a name for the service.
|
||||
func Name(val string) Option {
|
||||
return func(o *Options) {
|
||||
o.Name = val
|
||||
}
|
||||
}
|
||||
|
||||
// Address provides an address for the service.
|
||||
func Address(val string) Option {
|
||||
return func(o *Options) {
|
||||
o.Address = val
|
||||
}
|
||||
}
|
||||
|
||||
// Context provides a function to set the context option.
|
||||
func Context(val context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = val
|
||||
}
|
||||
}
|
||||
|
||||
// Config provides a function to set the config option.
|
||||
func Config(val *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = val
|
||||
}
|
||||
}
|
||||
|
||||
// Metrics provides a function to set the metrics option.
|
||||
func Metrics(val *metrics.Metrics) Option {
|
||||
return func(o *Options) {
|
||||
o.Metrics = val
|
||||
}
|
||||
}
|
||||
|
||||
// Namespace provides a function to set the namespace option.
|
||||
func Namespace(val string) Option {
|
||||
return func(o *Options) {
|
||||
o.Namespace = val
|
||||
}
|
||||
}
|
||||
|
||||
// Flags provides a function to set the flags option.
|
||||
func Flags(flags []cli.Flag) Option {
|
||||
return func(o *Options) {
|
||||
o.Flags = append(o.Flags, flags...)
|
||||
}
|
||||
}
|
||||
|
||||
// Store provides a function to configure the store
|
||||
func Store(store store.Store) Option {
|
||||
return func(o *Options) {
|
||||
o.Store = store
|
||||
}
|
||||
}
|
||||
|
||||
// Consumer provides a function to configure the consumer
|
||||
func Consumer(consumer events.Consumer) Option {
|
||||
return func(o *Options) {
|
||||
o.Consumer = consumer
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/version"
|
||||
ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
|
||||
svc "github.com/owncloud/ocis/v2/services/eventhistory/pkg/service"
|
||||
)
|
||||
|
||||
// NewService initializes the grpc service and server.
|
||||
func NewService(opts ...Option) grpc.Service {
|
||||
options := newOptions(opts...)
|
||||
|
||||
service, err := grpc.NewService(
|
||||
grpc.TLSEnabled(options.Config.GRPC.TLS.Enabled),
|
||||
grpc.TLSCert(
|
||||
options.Config.GRPC.TLS.Cert,
|
||||
options.Config.GRPC.TLS.Key,
|
||||
),
|
||||
grpc.Logger(options.Logger),
|
||||
grpc.Namespace(options.Namespace),
|
||||
grpc.Name(options.Name),
|
||||
grpc.Version(version.GetString()),
|
||||
grpc.Address(options.Address),
|
||||
grpc.Context(options.Context),
|
||||
grpc.Flags(options.Flags...),
|
||||
grpc.Version(version.GetString()),
|
||||
)
|
||||
if err != nil {
|
||||
options.Logger.Fatal().Err(err).Msg("Error creating event history service")
|
||||
return grpc.Service{}
|
||||
}
|
||||
|
||||
eh, err := svc.NewEventHistoryService(options.Config, options.Consumer, options.Store, options.Logger)
|
||||
if err != nil {
|
||||
options.Logger.Fatal().Err(err).Msg("Error creating event history service")
|
||||
return grpc.Service{}
|
||||
}
|
||||
|
||||
_ = ehsvc.RegisterEventHistoryServiceHandler(
|
||||
service.Server(),
|
||||
eh,
|
||||
)
|
||||
|
||||
return service
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/events"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
ehmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/eventhistory/v0"
|
||||
ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
|
||||
"go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
// EventHistoryService is the service responsible for event history
|
||||
type EventHistoryService struct {
|
||||
ch <-chan events.Event
|
||||
store store.Store
|
||||
cfg *config.Config
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
// NewEventHistoryService returns an EventHistory service
|
||||
func NewEventHistoryService(cfg *config.Config, consumer events.Consumer, store store.Store, log log.Logger) (*EventHistoryService, error) {
|
||||
if consumer == nil || store == nil {
|
||||
return nil, fmt.Errorf("Need non nil consumer (%v) and store (%v) to work properly", consumer, store)
|
||||
}
|
||||
|
||||
ch, err := events.ConsumeAll(consumer, "evhistory")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
eh := &EventHistoryService{ch: ch, store: store, cfg: cfg, log: log}
|
||||
go eh.StoreEvents()
|
||||
|
||||
return eh, nil
|
||||
}
|
||||
|
||||
// StoreEvents consumes all events and stores them in the store. Will block
|
||||
func (eh *EventHistoryService) StoreEvents() {
|
||||
for event := range eh.ch {
|
||||
if err := eh.store.Write(&store.Record{
|
||||
Key: event.ID,
|
||||
Value: event.Event.([]byte),
|
||||
Expiry: eh.cfg.Store.RecordExpiry,
|
||||
Metadata: map[string]interface{}{
|
||||
"type": event.Type,
|
||||
},
|
||||
}); err != nil {
|
||||
// we can't store. That's it for us.
|
||||
eh.log.Error().Err(err).Str("eventid", event.ID).Msg("could not store event")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetEvents allows to retrieve events from the eventstore by id
|
||||
func (eh *EventHistoryService) GetEvents(ctx context.Context, req *ehsvc.GetEventsRequest, resp *ehsvc.GetEventsResponse) error {
|
||||
for _, id := range req.Ids {
|
||||
evs, err := eh.store.Read(id)
|
||||
if err != nil {
|
||||
if err != store.ErrNotFound {
|
||||
eh.log.Error().Err(err).Str("eventid", id).Msg("could not read event")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
resp.Events = append(resp.Events, &ehmsg.Event{
|
||||
Id: id,
|
||||
Event: evs[0].Value,
|
||||
Type: evs[0].Metadata["type"].(string),
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestSearch(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Service Suite")
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/events"
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/log"
|
||||
"github.com/owncloud/ocis/v2/ocis-pkg/store"
|
||||
ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
|
||||
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/service"
|
||||
microevents "go-micro.dev/v4/events"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
var _ = Describe("EventHistoryService", func() {
|
||||
var (
|
||||
cfg = &config.Config{}
|
||||
|
||||
eh *service.EventHistoryService
|
||||
bus testBus
|
||||
sto microstore.Store
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
sto = store.Create()
|
||||
bus = testBus(make(chan events.Event))
|
||||
eh, err = service.NewEventHistoryService(cfg, bus, sto, log.Logger{})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
close(bus)
|
||||
})
|
||||
|
||||
It("Records events, stores them and allows to retrieve them", func() {
|
||||
id := bus.Publish(events.UploadReady{})
|
||||
|
||||
// service will store eventually
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
resp := &ehsvc.GetEventsResponse{}
|
||||
err := eh.GetEvents(context.Background(), &ehsvc.GetEventsRequest{Ids: []string{id}}, resp)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp).ToNot(BeNil())
|
||||
|
||||
Expect(len(resp.Events)).To(Equal(1))
|
||||
Expect(resp.Events[0].Id).To(Equal(id))
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
type testBus chan events.Event
|
||||
|
||||
func (tb testBus) Consume(_ string, _ ...microevents.ConsumeOption) (<-chan microevents.Event, error) {
|
||||
ch := make(chan microevents.Event)
|
||||
go func() {
|
||||
for ev := range tb {
|
||||
b, _ := json.Marshal(ev.Event)
|
||||
ch <- microevents.Event{
|
||||
Payload: b,
|
||||
Metadata: map[string]string{
|
||||
events.MetadatakeyEventID: ev.ID,
|
||||
events.MetadatakeyEventType: ev.Type,
|
||||
},
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (tb testBus) Publish(e interface{}) string {
|
||||
ev := events.Event{
|
||||
ID: uuid.New().String(),
|
||||
Type: reflect.TypeOf(e).String(),
|
||||
Event: e,
|
||||
}
|
||||
|
||||
tb <- ev
|
||||
return ev.ID
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@@ -156,10 +157,10 @@ func NewService(opts ...Option) (Graph, error) {
|
||||
|
||||
roleManager := options.RoleManager
|
||||
if roleManager == nil {
|
||||
storeOptions := store.OcisStoreOptions{
|
||||
Type: options.Config.CacheStore.Type,
|
||||
Address: options.Config.CacheStore.Address,
|
||||
Size: options.Config.CacheStore.Size,
|
||||
storeOptions := []store.Option{
|
||||
store.Type(options.Config.CacheStore.Type),
|
||||
store.Addresses(strings.Split(options.Config.CacheStore.Address, ",")...),
|
||||
store.Size(options.Config.CacheStore.Size),
|
||||
}
|
||||
m := roles.NewManager(
|
||||
roles.StoreOptions(storeOptions),
|
||||
|
||||
Reference in New Issue
Block a user