introduce eventhistory service

Signed-off-by: jkoberg <jkoberg@owncloud.com>
This commit is contained in:
jkoberg
2023-02-02 18:19:19 +01:00
parent 30faae2ae4
commit afe9e220b4
28 changed files with 1426 additions and 1 deletions
+37
View File
@@ -0,0 +1,37 @@
SHELL := bash
NAME := eventhistory
include ../../.make/recursion.mk
############ tooling ############
ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI
include ../../.bingo/Variables.mk
endif
############ go tooling ############
include ../../.make/go.mk
############ release ############
include ../../.make/release.mk
############ docs generate ############
include ../../.make/docs.mk
.PHONY: docs-generate
docs-generate: config-docs-generate
############ generate ############
include ../../.make/generate.mk
.PHONY: ci-go-generate
ci-go-generate: # CI runs ci-node-generate automatically before this target
.PHONY: ci-node-generate
ci-node-generate:
############ licenses ############
.PHONY: ci-node-check-licenses
ci-node-check-licenses:
.PHONY: ci-node-save-licenses
ci-node-save-licenses:
+15
View File
@@ -0,0 +1,15 @@
# Eventhistory service
The `eventhistory` consumes all events from the configured event systems, stores them and allows to retrieve them via an eventid
## Consuming
The `eventhistory` services consumes all events from the configured event sytem. Running it without an event sytem is not possible.
## Storing
The `eventhistory` stores each consumed event in the configured store. Possible stores are ? and ? but not ?.
## Retrieving
Other services can call the `eventhistory` service via a grpc call to retrieve events. The request must contain the eventid that should be retrieved
@@ -0,0 +1,14 @@
package main
import (
"os"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/command"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config/defaults"
)
func main() {
if err := command.Execute(defaults.DefaultConfig()); err != nil {
os.Exit(1)
}
}
@@ -0,0 +1,18 @@
package command
import (
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
"github.com/urfave/cli/v2"
)
// Health is the entrypoint for the health command.
func Health(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "health",
Usage: "Check health status",
Action: func(c *cli.Context) error {
// Not implemented
return nil
},
}
}
+59
View File
@@ -0,0 +1,59 @@
package command
import (
"context"
"os"
"github.com/owncloud/ocis/v2/ocis-pkg/clihelper"
ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// GetCommands provides all commands for this service
func GetCommands(cfg *config.Config) cli.Commands {
return []*cli.Command{
// start this service
Server(cfg),
// interaction with this service
// infos about this service
Health(cfg),
Version(cfg),
}
}
// Execute is the entry point for the eventhistory command.
func Execute(cfg *config.Config) error {
app := clihelper.DefaultApp(&cli.App{
Name: "eventhistory",
Usage: "starts eventhistory service",
Commands: GetCommands(cfg),
})
return app.Run(os.Args)
}
// SutureService allows for the eventhistory command to be embedded and supervised by a suture supervisor tree.
type SutureService struct {
cfg *config.Config
}
// NewSutureService creates a new eventhistory.SutureService
func NewSutureService(cfg *ociscfg.Config) suture.Service {
cfg.Notifications.Commons = cfg.Commons
return SutureService{
//cfg: cfg.Notifications,
}
}
func (s SutureService) Serve(ctx context.Context) error {
s.cfg.Context = ctx
if err := Execute(s.cfg); err != nil {
return err
}
return nil
}
@@ -0,0 +1,85 @@
package command
import (
"context"
"fmt"
"github.com/cs3org/reva/v2/pkg/events/stream"
"github.com/oklog/run"
"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
"github.com/owncloud/ocis/v2/ocis-pkg/version"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config/parser"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/logging"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/metrics"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/server/grpc"
"github.com/urfave/cli/v2"
"go-micro.dev/v4/store"
)
// Server is the entrypoint for the server command.
func Server(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "server",
Usage: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name),
Category: "server",
Before: func(c *cli.Context) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
Action: func(c *cli.Context) error {
logger := logging.Configure(cfg.Service.Name, cfg.Log)
err := ogrpc.Configure(ogrpc.GetClientOptions(cfg.GRPCClientTLS)...)
if err != nil {
return err
}
var (
gr = run.Group{}
ctx, cancel = func() (context.Context, context.CancelFunc) {
if cfg.Context == nil {
return context.WithCancel(context.Background())
}
return context.WithCancel(cfg.Context)
}()
metrics = metrics.New()
)
defer cancel()
metrics.BuildInfo.WithLabelValues(version.GetString()).Set(1)
consumer, err := stream.NatsFromConfig(stream.NatsConfig(cfg.Events))
if err != nil {
return err
}
// TODO: configure store
st := store.DefaultStore
service := grpc.NewService(
grpc.Logger(logger),
grpc.Context(ctx),
grpc.Config(cfg),
grpc.Name(cfg.Service.Name),
grpc.Namespace(cfg.GRPC.Namespace),
grpc.Address(cfg.GRPC.Addr),
grpc.Metrics(metrics),
grpc.Consumer(consumer),
grpc.Store(st),
)
gr.Add(service.Run, func(_ error) {
logger.Error().
Err(err).
Str("server", "grpc").
Msg("Shutting down server")
cancel()
})
return gr.Run()
},
}
}
@@ -0,0 +1,19 @@
package command
import (
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
"github.com/urfave/cli/v2"
)
// Version prints the service versions of all running instances.
func Version(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "version",
Usage: "print the version of this binary and the running service instances",
Category: "info",
Action: func(c *cli.Context) error {
// not implemented
return nil
},
}
}
@@ -0,0 +1,47 @@
package config
import (
"context"
"time"
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
)
// Config combines all available configuration parts.
type Config struct {
Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service
Service Service `yaml:"-"`
Log *Log `yaml:"log"`
Debug Debug `yaml:"debug"`
GRPC GRPCConfig `yaml:"grpc"`
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
Events Events `yaml:"events"`
Store Store `yaml:"store"`
Context context.Context `yaml:"-"`
}
// GRPCConfig defines the available grpc configuration.
type GRPCConfig struct {
Addr string `ocisConfig:"addr" env:"EVENTHISTORY_GRPC_ADDR" desc:"The bind address of the GRPC service."`
Namespace string `ocisConfig:"-" yaml:"-"`
TLS *shared.GRPCServiceTLS `yaml:"tls"`
}
// Store configures the store to use
type Store struct {
RecordExpiry time.Duration `yaml:"record_expiry" env:"RECORD_EXPIRY" desc:"time to life for events in the store"`
}
// Events combines the configuration options for the event bus.
type Events struct {
Endpoint string `yaml:"endpoint" env:"EVENTHISTORY_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture."`
Cluster string `yaml:"cluster" env:"EVENTHISTORY_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system."`
TLSInsecure bool `yaml:"tls_insecure" env:"OCIS_INSECURE;EVENTHISTORY_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates."`
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"EVENTHISTORY_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided NOTIFICATIONS_EVENTS_TLS_INSECURE will be seen as false."`
EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS;EVENTHISTORY_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services.."`
}
@@ -0,0 +1,9 @@
package config
// Debug defines the available debug configuration.
type Debug struct {
Addr string `yaml:"addr" env:"EVENTHISTORY_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed."`
Token string `yaml:"token" env:"EVENTHISTORY_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint."`
Pprof bool `yaml:"pprof" env:"EVENTHISTORY_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling."`
Zpages bool `yaml:"zpages" env:"EVENTHISTORY_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces."`
}
@@ -0,0 +1,50 @@
package defaults
import (
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
)
// FullDefaultConfig returns the full default config
func FullDefaultConfig() *config.Config {
cfg := DefaultConfig()
EnsureDefaults(cfg)
Sanitize(cfg)
return cfg
}
// DefaultConfig return the default configuration
func DefaultConfig() *config.Config {
return &config.Config{
Service: config.Service{
Name: "eventhistory",
},
}
}
// EnsureDefaults ensures the config contains default values
func EnsureDefaults(cfg *config.Config) {
// provide with defaults for shared logging, since we need a valid destination address for "envdecode".
if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
cfg.Log = &config.Log{
Level: cfg.Commons.Log.Level,
Pretty: cfg.Commons.Log.Pretty,
Color: cfg.Commons.Log.Color,
File: cfg.Commons.Log.File,
}
} else if cfg.Log == nil {
cfg.Log = &config.Log{}
}
if cfg.GRPCClientTLS == nil {
cfg.GRPCClientTLS = &shared.GRPCClientTLS{}
if cfg.Commons != nil && cfg.Commons.GRPCClientTLS != nil {
cfg.GRPCClientTLS = cfg.Commons.GRPCClientTLS
}
}
}
// Sanitize sanitizes the config
func Sanitize(cfg *config.Config) {
// nothing to sanitize here atm
}
+9
View File
@@ -0,0 +1,9 @@
package config
// Log defines the available log configuration.
type Log struct {
Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;EVENTHISTORY_LOG_LEVEL" desc:"The log level. Valid values are: \"panic\", \"fatal\", \"error\", \"warn\", \"info\", \"debug\", \"trace\"."`
Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;EVENTHISTORY_LOG_PRETTY" desc:"Activates pretty log output."`
Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;EVENTHISTORY_LOG_COLOR" desc:"Activates colorized log output."`
File string `mapstructure:"file" env:"OCIS_LOG_FILE;EVENTHISTORY_LOG_FILE" desc:"The path to the log file. Activates logging to this file if set."`
}
@@ -0,0 +1,38 @@
package parser
import (
"errors"
ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config/defaults"
"github.com/owncloud/ocis/v2/ocis-pkg/config/envdecode"
)
// ParseConfig loads configuration from known paths.
func ParseConfig(cfg *config.Config) error {
_, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg)
if err != nil {
return err
}
defaults.EnsureDefaults(cfg)
// load all env variables relevant to the config in the current context.
if err := envdecode.Decode(cfg); err != nil {
// no environment variable set for this config is an expected "error"
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
return err
}
}
defaults.Sanitize(cfg)
return Validate(cfg)
}
// Validate validates the config
func Validate(cfg *config.Config) error {
return nil
}
@@ -0,0 +1,6 @@
package config
// Service defines the available service configuration.
type Service struct {
Name string `yaml:"-"`
}
@@ -0,0 +1,17 @@
package logging
import (
"github.com/owncloud/ocis/v2/ocis-pkg/log"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
)
// LoggerFromConfig initializes a service-specific logger instance.
func Configure(name string, cfg *config.Log) log.Logger {
return log.NewLogger(
log.Name(name),
log.Level(cfg.Level),
log.Pretty(cfg.Pretty),
log.Color(cfg.Color),
log.File(cfg.File),
)
}
@@ -0,0 +1,35 @@
package metrics
import "github.com/prometheus/client_golang/prometheus"
var (
// Namespace defines the namespace for the defines metrics.
Namespace = "ocis"
// Subsystem defines the subsystem for the defines metrics.
Subsystem = "eventhistory"
)
// Metrics defines the available metrics of this service.
type Metrics struct {
BuildInfo *prometheus.GaugeVec
}
// New initializes the available metrics.
func New() *Metrics {
m := &Metrics{
BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: Namespace,
Subsystem: Subsystem,
Name: "build_info",
Help: "Build information",
}, []string{"version"}),
}
_ = prometheus.Register(
m.BuildInfo,
)
// TODO: implement metrics
return m
}
@@ -0,0 +1,110 @@
package grpc
import (
"context"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/metrics"
"github.com/urfave/cli/v2"
"go-micro.dev/v4/store"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Name string
Address string
Logger log.Logger
Context context.Context
Config *config.Config
Metrics *metrics.Metrics
Namespace string
Flags []cli.Flag
Store store.Store
Consumer events.Consumer
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Name provides a name for the service.
func Name(val string) Option {
return func(o *Options) {
o.Name = val
}
}
// Address provides an address for the service.
func Address(val string) Option {
return func(o *Options) {
o.Address = val
}
}
// Context provides a function to set the context option.
func Context(val context.Context) Option {
return func(o *Options) {
o.Context = val
}
}
// Config provides a function to set the config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
// Metrics provides a function to set the metrics option.
func Metrics(val *metrics.Metrics) Option {
return func(o *Options) {
o.Metrics = val
}
}
// Namespace provides a function to set the namespace option.
func Namespace(val string) Option {
return func(o *Options) {
o.Namespace = val
}
}
// Flags provides a function to set the flags option.
func Flags(flags []cli.Flag) Option {
return func(o *Options) {
o.Flags = append(o.Flags, flags...)
}
}
// Store provides a function to configure the store
func Store(store store.Store) Option {
return func(o *Options) {
o.Store = store
}
}
// Consumer provides a function to configure the consumer
func Consumer(consumer events.Consumer) Option {
return func(o *Options) {
o.Consumer = consumer
}
}
@@ -0,0 +1,46 @@
package grpc
import (
"github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
"github.com/owncloud/ocis/v2/ocis-pkg/version"
ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
svc "github.com/owncloud/ocis/v2/services/eventhistory/pkg/service"
)
// NewService initializes the grpc service and server.
func NewService(opts ...Option) grpc.Service {
options := newOptions(opts...)
service, err := grpc.NewService(
grpc.TLSEnabled(options.Config.GRPC.TLS.Enabled),
grpc.TLSCert(
options.Config.GRPC.TLS.Cert,
options.Config.GRPC.TLS.Key,
),
grpc.Logger(options.Logger),
grpc.Namespace(options.Namespace),
grpc.Name(options.Name),
grpc.Version(version.GetString()),
grpc.Address(options.Address),
grpc.Context(options.Context),
grpc.Flags(options.Flags...),
grpc.Version(version.GetString()),
)
if err != nil {
options.Logger.Fatal().Err(err).Msg("Error creating event history service")
return grpc.Service{}
}
eh, err := svc.NewEventHistoryService(options.Config, options.Consumer, options.Store)
if err != nil {
options.Logger.Fatal().Err(err).Msg("Error creating event history service")
return grpc.Service{}
}
_ = ehsvc.RegisterEventHistoryServiceHandler(
service.Server(),
eh,
)
return service
}
@@ -0,0 +1,70 @@
package service
import (
"context"
"fmt"
"github.com/cs3org/reva/v2/pkg/events"
ehmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/eventhistory/v0"
ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
"github.com/owncloud/ocis/v2/services/eventhistory/pkg/config"
"go-micro.dev/v4/store"
)
// EventHistoryService is the service responsible for event history
type EventHistoryService struct {
ch <-chan events.Event
store store.Store
cfg *config.Config
}
// NewEventHistoryService returns an EventHistory service
func NewEventHistoryService(cfg *config.Config, consumer events.Consumer, store store.Store) (*EventHistoryService, error) {
if consumer == nil || store == nil {
return nil, fmt.Errorf("Need non nil consumer (%v) and store (%v) to work properly", consumer, store)
}
ch, err := events.ConsumeAll(consumer, "evhistory")
if err != nil {
return nil, err
}
eh := &EventHistoryService{ch: ch, store: store, cfg: cfg}
go eh.StoreEvents()
return eh, nil
}
// StoreEvents consumes all events and stores them in the store. Will block
func (eh *EventHistoryService) StoreEvents() {
for event := range eh.ch {
if err := eh.store.Write(&store.Record{
Key: event.ID,
Value: event.Event.([]byte),
Expiry: eh.cfg.Store.RecordExpiry,
}); err != nil {
// we can't store. That's it for us.
return
}
}
}
// GetEvents allows to retrieve events from the eventstore by id
func (eh *EventHistoryService) GetEvents(ctx context.Context, req *ehsvc.GetEventsRequest, resp *ehsvc.GetEventsResponse) error {
for _, id := range req.Ids {
evs, err := eh.store.Read(id)
if err != nil {
// TODO: Handle!
// return?
// gather errors and add to response?
continue
}
resp.Events = append(resp.Events, &ehmsg.Event{
Id: id,
Event: evs[0].Value,
})
}
return nil
}
@@ -0,0 +1,3 @@
package service_test
// tests here