Merge pull request #9327 from kobergj/Activities

Activitylog Service
This commit is contained in:
kobergj
2024-06-10 12:28:14 +02:00
committed by GitHub
27 changed files with 1115 additions and 3 deletions

View File

@@ -20,6 +20,7 @@ L10N_MODULES := \
# if you add a module here please also add it to the .drone.star file
OCIS_MODULES = \
services/activitylog \
services/antivirus \
services/app-provider \
services/app-registry \

View File

@@ -0,0 +1,5 @@
Enhancement: Activitylog Service
Adds a new service `activitylog` which stores events (activities) per resource. This data can be retrieved by clients to show item activities
https://github.com/owncloud/ocis/pull/9327

View File

@@ -49,7 +49,7 @@ We also suggest using the last port in your extensions' range as a debug/metrics
| 9180-9184 | FREE (formerly used by accounts) |
| 9185-9189 | [thumbnails]({{< ref "../thumbnails/_index.md" >}}) |
| 9190-9194 | [settings]({{< ref "../settings/_index.md" >}}) |
| 9195-9197 | FREE |
| 9195-9197 | [activitylog]({{< ref "../activitylog/_index.md" >}}) |
| 9198-9199 | [auth-service]({{< ref "../auth-service/_index.md" >}}) |
| 9200-9204 | [proxy]({{< ref "../proxy/_index.md" >}}) |
| 9205-9209 | [proxy]({{< ref "../proxy/_index.md" >}}) |
@@ -63,11 +63,11 @@ We also suggest using the last port in your extensions' range as a debug/metrics
| 9245-9249 | FREE |
| 9250-9254 | [ocis server (runtime)](https://github.com/owncloud/ocis/tree/master/ocis/pkg/runtime) |
| 9255-9259 | [postprocessing]({{< ref "../postprocessing/_index.md" >}}) |
| 9260-9264 | [clientlog]({{< ref "../clientlog/_index.md" >}}) |
| 9260-9264 | [clientlog]({{< ref "../clientlog/_index.md" >}}) |
| 9265-9269 | FREE |
| 9270-9274 | [eventhistory]({{< ref "../eventhistory/_index.md" >}}) |
| 9275-9279 | FREE |
| 9280-9284 | [ocm]({{< ref "../ocm/_index.md" >}}) |
| 9280-9284 | [ocm]({{< ref "../ocm/_index.md" >}}) |
| 9285-9289 | FREE |
| 9290-9294 | FREE |
| 9295-9299 | FREE |

View File

@@ -2,6 +2,7 @@ package config
import (
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
activitylog "github.com/owncloud/ocis/v2/services/activitylog/pkg/config"
antivirus "github.com/owncloud/ocis/v2/services/antivirus/pkg/config"
appProvider "github.com/owncloud/ocis/v2/services/app-provider/pkg/config"
appRegistry "github.com/owncloud/ocis/v2/services/app-registry/pkg/config"
@@ -80,6 +81,7 @@ type Config struct {
AdminUserID string `yaml:"admin_user_id" env:"OCIS_ADMIN_USER_ID" desc:"ID of a user, that should receive admin privileges. Consider that the UUID can be encoded in some LDAP deployment configurations like in .ldif files. These need to be decoded beforehand." introductionVersion:"pre5.0"`
Runtime Runtime `yaml:"runtime"`
Activitylog *activitylog.Config `yaml:"activitylog"`
Antivirus *antivirus.Config `yaml:"antivirus"`
AppProvider *appProvider.Config `yaml:"app_provider"`
AppRegistry *appRegistry.Config `yaml:"app_registry"`

View File

@@ -1,6 +1,7 @@
package config
import (
activitylog "github.com/owncloud/ocis/v2/services/activitylog/pkg/config/defaults"
antivirus "github.com/owncloud/ocis/v2/services/antivirus/pkg/config/defaults"
appProvider "github.com/owncloud/ocis/v2/services/app-provider/pkg/config/defaults"
appRegistry "github.com/owncloud/ocis/v2/services/app-registry/pkg/config/defaults"
@@ -52,6 +53,7 @@ func DefaultConfig() *Config {
Host: "localhost",
},
Activitylog: activitylog.DefaultConfig(),
Antivirus: antivirus.DefaultConfig(),
AppProvider: appProvider.DefaultConfig(),
AppRegistry: appRegistry.DefaultConfig(),

View File

@@ -8,6 +8,7 @@ import (
"github.com/owncloud/ocis/v2/ocis-pkg/config/parser"
"github.com/owncloud/ocis/v2/ocis/pkg/command/helper"
"github.com/owncloud/ocis/v2/ocis/pkg/register"
activitylog "github.com/owncloud/ocis/v2/services/activitylog/pkg/command"
antivirus "github.com/owncloud/ocis/v2/services/antivirus/pkg/command"
appprovider "github.com/owncloud/ocis/v2/services/app-provider/pkg/command"
appregistry "github.com/owncloud/ocis/v2/services/app-registry/pkg/command"
@@ -52,6 +53,11 @@ import (
)
var svccmds = []register.Command{
func(cfg *config.Config) *cli.Command {
return ServiceCommand(cfg, cfg.Activitylog.Service.Name, activitylog.GetCommands(cfg.Activitylog), func(c *config.Config) {
cfg.Activitylog.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cli.Command {
return ServiceCommand(cfg, cfg.Antivirus.Service.Name, antivirus.GetCommands(cfg.Antivirus), func(c *config.Config) {
// cfg.Antivirus.Commons = cfg.Commons // antivirus needs no commons atm

View File

@@ -172,6 +172,11 @@ type Nats struct {
}
}
// Activitylog is the configuration for the activitylog service
type Activitylog struct {
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// ServiceAccount is the configuration for the used service account
type ServiceAccount struct {
ServiceAccountID string `yaml:"service_account_id"`
@@ -221,6 +226,7 @@ type OcisConfig struct {
Userlog Userlog
AuthService AuthService `yaml:"auth_service"`
Clientlog Clientlog
Activitylog Activitylog
}
func checkConfigPath(configPath string) error {
@@ -429,6 +435,9 @@ func CreateConfig(insecure, forceOverwrite bool, configPath, adminPassword strin
Settings: SettingsService{
ServiceAccountIDs: []string{serviceAccount.ServiceAccountID},
},
Activitylog: Activitylog{
ServiceAccount: serviceAccount,
},
}
if insecure {

View File

@@ -25,6 +25,7 @@ import (
"github.com/owncloud/ocis/v2/ocis-pkg/log"
ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
activitylog "github.com/owncloud/ocis/v2/services/activitylog/pkg/command"
antivirus "github.com/owncloud/ocis/v2/services/antivirus/pkg/command"
appProvider "github.com/owncloud/ocis/v2/services/app-provider/pkg/command"
appRegistry "github.com/owncloud/ocis/v2/services/app-registry/pkg/command"
@@ -144,6 +145,11 @@ func NewService(options ...Option) (*Service, error) {
// priority group 2 is empty for now
// most services are in priority group 3
reg(3, opts.Config.Activitylog.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
cfg.Activitylog.Context = ctx
cfg.Activitylog.Commons = cfg.Commons
return activitylog.Execute(cfg.Activitylog)
})
reg(3, opts.Config.AppProvider.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
cfg.AppProvider.Context = ctx
cfg.AppProvider.Commons = cfg.Commons

View File

@@ -0,0 +1,37 @@
SHELL := bash
NAME := activitylog
include ../../.make/recursion.mk
############ tooling ############
ifneq (, $(shell command -v go 2> /dev/null)) # suppress "command not found warnings" for non go targets in CI
include ../../.bingo/Variables.mk
endif
############ go tooling ############
include ../../.make/go.mk
############ release ############
include ../../.make/release.mk
############ docs generate ############
include ../../.make/docs.mk
.PHONY: docs-generate
docs-generate: config-docs-generate
############ generate ############
include ../../.make/generate.mk
.PHONY: ci-go-generate
ci-go-generate: # CI runs ci-node-generate automatically before this target
.PHONY: ci-node-generate
ci-node-generate:
############ licenses ############
.PHONY: ci-node-check-licenses
ci-node-check-licenses:
.PHONY: ci-node-save-licenses
ci-node-save-licenses:

View File

@@ -0,0 +1,15 @@
# Activitylog Service
The `activitylog` service is responsible for storing events (activities) per resource.
## The Log Service Ecosystem
Log services like the `activitylog`, `userlog`, `clientlog` and `sse` are responsible for composing notifications for a specific audience.
- The `userlog` service translates and adjusts messages to be human readable.
- The `clientlog` service composes machine readable messages, so clients can act without the need to query the server.
- The `sse` service is only responsible for sending these messages. It does not care about their form or language.
- The `activitylog` service stores events per resource. These can be retrieved to show item activities
## Activitylog Store
The `activitylog` stores activities for each resource. It works in conjunction with the `eventhistory` service to keep the data it needs to store to a minimum.

View File

@@ -0,0 +1,14 @@
package main
import (
"os"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/command"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config/defaults"
)
func main() {
if err := command.Execute(defaults.DefaultConfig()); err != nil {
os.Exit(1)
}
}

View File

@@ -0,0 +1,18 @@
package command
import (
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config"
"github.com/urfave/cli/v2"
)
// Health is the entrypoint for the health command.
func Health(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "health",
Usage: "Check health status",
Action: func(c *cli.Context) error {
// Not implemented
return nil
},
}
}

View File

@@ -0,0 +1,34 @@
package command
import (
"os"
"github.com/owncloud/ocis/v2/ocis-pkg/clihelper"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config"
"github.com/urfave/cli/v2"
)
// GetCommands provides all commands for this service
func GetCommands(cfg *config.Config) cli.Commands {
return []*cli.Command{
// start this service
Server(cfg),
// interaction with this service
// infos about this service
Health(cfg),
Version(cfg),
}
}
// Execute is the entry point for the activitylog command.
func Execute(cfg *config.Config) error {
app := clihelper.DefaultApp(&cli.App{
Name: "activitylog",
Usage: "starts activitylog service",
Commands: GetCommands(cfg),
})
return app.Run(os.Args)
}

View File

@@ -0,0 +1,163 @@
package command
import (
"context"
"fmt"
"os"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/events/stream"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/store"
"github.com/oklog/run"
"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
"github.com/owncloud/ocis/v2/ocis-pkg/handlers"
"github.com/owncloud/ocis/v2/ocis-pkg/registry"
"github.com/owncloud/ocis/v2/ocis-pkg/service/debug"
"github.com/owncloud/ocis/v2/ocis-pkg/tracing"
"github.com/owncloud/ocis/v2/ocis-pkg/version"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config/parser"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/logging"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/metrics"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/service"
"github.com/urfave/cli/v2"
microstore "go-micro.dev/v4/store"
)
var _registeredEvents = []events.Unmarshaller{
events.UploadReady{},
events.FileTouched{},
events.ContainerCreated{},
events.ItemTrashed{},
events.ItemPurged{},
events.ItemMoved{},
events.ShareCreated{},
events.ShareUpdated{},
events.ShareRemoved{},
events.LinkCreated{},
events.LinkUpdated{},
events.LinkRemoved{},
events.SpaceShared{},
events.SpaceShareUpdated{},
events.SpaceUnshared{},
// TODO: file downloaded only for public links. How to do this?
events.FileDownloaded{},
}
// Server is the entrypoint for the server command.
func Server(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "server",
Usage: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name),
Category: "server",
Before: func(c *cli.Context) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
Action: func(c *cli.Context) error {
logger := logging.Configure(cfg.Service.Name, cfg.Log)
tracerProvider, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name)
if err != nil {
logger.Error().Err(err).Msg("Failed to initialize tracer")
return err
}
gr := run.Group{}
ctx, cancel := func() (context.Context, context.CancelFunc) {
if cfg.Context == nil {
return context.WithCancel(context.Background())
}
return context.WithCancel(cfg.Context)
}()
mtrcs := metrics.New()
mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1)
defer cancel()
evStream, err := stream.NatsFromConfig(cfg.Service.Name, false, stream.NatsConfig(cfg.Events))
if err != nil {
logger.Error().Err(err).Msg("Failed to initialize event stream")
return err
}
evStore := store.Create(
store.Store(cfg.Store.Store),
store.TTL(cfg.Store.TTL),
store.Size(cfg.Store.Size),
microstore.Nodes(cfg.Store.Nodes...),
microstore.Database(cfg.Store.Database),
microstore.Table(cfg.Store.Table),
store.Authentication(cfg.Store.AuthUsername, cfg.Store.AuthPassword),
)
tm, err := pool.StringToTLSMode(cfg.GRPCClientTLS.Mode)
if err != nil {
logger.Error().Err(err).Msg("Failed to parse tls mode")
return err
}
gatewaySelector, err := pool.GatewaySelector(
cfg.RevaGateway,
pool.WithTLSCACert(cfg.GRPCClientTLS.CACert),
pool.WithTLSMode(tm),
pool.WithRegistry(registry.GetRegistry()),
pool.WithTracerProvider(tracerProvider),
)
if err != nil {
logger.Error().Err(err).Msg("Failed to initialize gateway selector")
return fmt.Errorf("could not get reva client selector: %s", err)
}
{
svc, err := service.New(
service.Logger(logger),
service.Config(cfg),
service.TraceProvider(tracerProvider),
service.Stream(evStream),
service.RegisteredEvents(_registeredEvents),
service.Store(evStore),
service.GatewaySelector(gatewaySelector),
)
if err != nil {
logger.Error().Err(err).Str("transport", "http").Msg("Failed to initialize server")
return err
}
gr.Add(func() error {
return svc.Run()
}, func(err error) {
logger.Error().
Str("transport", "http").
Err(err).
Msg("Shutting down server")
cancel()
os.Exit(1)
})
}
{
server := debug.NewService(
debug.Logger(logger),
debug.Name(cfg.Service.Name),
debug.Version(version.GetString()),
debug.Address(cfg.Debug.Addr),
debug.Token(cfg.Debug.Token),
debug.Pprof(cfg.Debug.Pprof),
debug.Zpages(cfg.Debug.Zpages),
debug.Health(handlers.Health),
debug.Ready(handlers.Ready),
)
gr.Add(server.ListenAndServe, func(_ error) {
_ = server.Shutdown(ctx)
cancel()
})
}
return gr.Run()
},
}
}

View File

@@ -0,0 +1,19 @@
package command
import (
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config"
"github.com/urfave/cli/v2"
)
// Version prints the service versions of all running instances.
func Version(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "version",
Usage: "print the version of this binary and the running service instances",
Category: "info",
Action: func(c *cli.Context) error {
// not implemented
return nil
},
}
}

View File

@@ -0,0 +1,58 @@
package config
import (
"context"
"time"
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
)
// Config combines all available configuration parts.
type Config struct {
Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service
Service Service `yaml:"-"`
Tracing *Tracing `yaml:"tracing"`
Log *Log `yaml:"log"`
Debug Debug `yaml:"debug"`
Events Events `yaml:"events"`
Store Store `yaml:"store"`
RevaGateway string `yaml:"reva_gateway" env:"OCIS_REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata" introductionVersion:"5.0"`
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
ServiceAccount ServiceAccount `yaml:"service_account"`
Context context.Context `yaml:"-"`
}
// Events combines the configuration options for the event bus.
type Events struct {
Endpoint string `yaml:"endpoint" env:"OCIS_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"5.0"`
Cluster string `yaml:"cluster" env:"OCIS_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"5.0"`
TLSInsecure bool `yaml:"tls_insecure" env:"OCIS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"5.0"`
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OCIS_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided NOTIFICATIONS_EVENTS_TLS_INSECURE will be seen as false." introductionVersion:"5.0"`
EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services." introductionVersion:"5.0"`
AuthUsername string `yaml:"username" env:"OCIS_EVENTS_AUTH_USERNAME" desc:"The username to authenticate with the events broker. The events broker is the ocis service which receives and delivers events between the services." introductionVersion:"5.0"`
AuthPassword string `yaml:"password" env:"OCIS_EVENTS_AUTH_PASSWORD" desc:"The password to authenticate with the events broker. The events broker is the ocis service which receives and delivers events between the services." introductionVersion:"5.0"`
}
// Store configures the store to use
type Store struct {
Store string `yaml:"store" env:"OCIS_PERSISTENT_STORE;ACTIVITYLOG_STORE" desc:"The type of the store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details." introductionVersion:"pre5.0"`
Nodes []string `yaml:"nodes" env:"OCIS_PERSISTENT_STORE_NODES;ACTIVITYLOG_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"`
Database string `yaml:"database" env:"ACTIVITYLOG_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"pre5.0"`
Table string `yaml:"table" env:"ACTIVITYLOG_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"pre5.0"`
TTL time.Duration `yaml:"ttl" env:"OCIS_PERSISTENT_STORE_TTL;ACTIVITYLOG_STORE_TTL" desc:"Time to live for events in the store. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"`
Size int `yaml:"size" env:"OCIS_PERSISTENT_STORE_SIZE;ACTIVITYLOG_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured. Defaults to 512 which is derived from the ocmem package though not exclicitely set as default." introductionVersion:"pre5.0"`
AuthUsername string `yaml:"username" env:"OCIS_PERSISTENT_STORE_AUTH_USERNAME;ACTIVITYLOG_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"`
AuthPassword string `yaml:"password" env:"OCIS_PERSISTENT_STORE_AUTH_PASSWORD;ACTIVITYLOG_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"5.0"`
}
// ServiceAccount is the configuration for the used service account
type ServiceAccount struct {
ServiceAccountID string `yaml:"service_account_id" env:"OCIS_SERVICE_ACCOUNT_ID;ACTIVITYLOG_SERVICE_ACCOUNT_ID" desc:"The ID of the service account the service should use. See the 'auth-service' service description for more details." introductionVersion:"5.0"`
ServiceAccountSecret string `yaml:"service_account_secret" env:"OCIS_SERVICE_ACCOUNT_SECRET;ACTIVITYOG_SERVICE_ACCOUNT_SECRET" desc:"The service account secret." introductionVersion:"5.0"`
}

View File

@@ -0,0 +1,9 @@
package config
// Debug defines the available debug configuration.
type Debug struct {
Addr string `yaml:"addr" env:"CLIENTLOG_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed." introductionVersion:"5.0"`
Token string `yaml:"token" env:"CLIENTLOG_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint." introductionVersion:"5.0"`
Pprof bool `yaml:"pprof" env:"CLIENTLOG_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling." introductionVersion:"5.0"`
Zpages bool `yaml:"zpages" env:"CLIENTLOG_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces." introductionVersion:"5.0"`
}

View File

@@ -0,0 +1,78 @@
package defaults
import (
"github.com/owncloud/ocis/v2/ocis-pkg/shared"
"github.com/owncloud/ocis/v2/ocis-pkg/structs"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config"
)
// FullDefaultConfig returns the full default config
func FullDefaultConfig() *config.Config {
cfg := DefaultConfig()
EnsureDefaults(cfg)
Sanitize(cfg)
return cfg
}
// DefaultConfig return the default configuration
func DefaultConfig() *config.Config {
return &config.Config{
Debug: config.Debug{
Addr: "127.0.0.1:9197",
Token: "",
Pprof: false,
Zpages: false,
},
Service: config.Service{
Name: "activitylog",
},
Events: config.Events{
Endpoint: "127.0.0.1:9233",
Cluster: "ocis-cluster",
EnableTLS: false,
},
Store: config.Store{
Store: "nats-js-kv",
Nodes: []string{"127.0.0.1:9233"},
Database: "activitylog",
Table: "",
},
RevaGateway: shared.DefaultRevaConfig().Address,
}
}
// EnsureDefaults ensures the config contains default values
func EnsureDefaults(cfg *config.Config) {
// provide with defaults for shared logging, since we need a valid destination address for "envdecode".
if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
cfg.Log = &config.Log{
Level: cfg.Commons.Log.Level,
Pretty: cfg.Commons.Log.Pretty,
Color: cfg.Commons.Log.Color,
File: cfg.Commons.Log.File,
}
} else if cfg.Log == nil {
cfg.Log = &config.Log{}
}
// provide with defaults for shared tracing, since we need a valid destination address for "envdecode".
if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil {
cfg.Tracing = &config.Tracing{
Enabled: cfg.Commons.Tracing.Enabled,
Type: cfg.Commons.Tracing.Type,
Endpoint: cfg.Commons.Tracing.Endpoint,
Collector: cfg.Commons.Tracing.Collector,
}
} else if cfg.Tracing == nil {
cfg.Tracing = &config.Tracing{}
}
if cfg.GRPCClientTLS == nil && cfg.Commons != nil {
cfg.GRPCClientTLS = structs.CopyOrZeroValue(cfg.Commons.GRPCClientTLS)
}
}
// Sanitize sanitizes the config
func Sanitize(cfg *config.Config) {
// sanitize config
}

View File

@@ -0,0 +1,9 @@
package config
// Log defines the available log configuration.
type Log struct {
Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;CLIENTLOG_USERLOG_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'." introductionVersion:"5.0"`
Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;CLIENTLOG_USERLOG_LOG_PRETTY" desc:"Activates pretty log output." introductionVersion:"5.0"`
Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;CLIENTLOG_USERLOG_LOG_COLOR" desc:"Activates colorized log output." introductionVersion:"5.0"`
File string `mapstructure:"file" env:"OCIS_LOG_FILE;CLIENTLOG_USERLOG_LOG_FILE" desc:"The path to the log file. Activates logging to this file if set." introductionVersion:"5.0"`
}

View File

@@ -0,0 +1,38 @@
package parser
import (
"errors"
ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config/defaults"
"github.com/owncloud/ocis/v2/ocis-pkg/config/envdecode"
)
// ParseConfig loads configuration from known paths.
func ParseConfig(cfg *config.Config) error {
err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg)
if err != nil {
return err
}
defaults.EnsureDefaults(cfg)
// load all env variables relevant to the config in the current context.
if err := envdecode.Decode(cfg); err != nil {
// no environment variable set for this config is an expected "error"
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
return err
}
}
defaults.Sanitize(cfg)
return Validate(cfg)
}
// Validate validates the config
func Validate(cfg *config.Config) error {
return nil
}

View File

@@ -0,0 +1,6 @@
package config
// Service defines the available service configuration.
type Service struct {
Name string `yaml:"-"`
}

View File

@@ -0,0 +1,21 @@
package config
import "github.com/owncloud/ocis/v2/ocis-pkg/tracing"
// Tracing defines the available tracing configuration.
type Tracing struct {
Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;CLIENTLOG_TRACING_ENABLED" desc:"Activates tracing." introductionVersion:"5.0"`
Type string `yaml:"type" env:"OCIS_TRACING_TYPE;CLIENTLOG_TRACING_TYPE" desc:"The type of tracing. Defaults to '', which is the same as 'jaeger'. Allowed tracing types are 'jaeger' and '' as of now." introductionVersion:"5.0"`
Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;CLIENTLOG_TRACING_ENDPOINT" desc:"The endpoint of the tracing agent." introductionVersion:"5.0"`
Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;CLIENTLOG_TRACING_COLLECTOR" desc:"The HTTP endpoint for sending spans directly to a collector, i.e. http://jaeger-collector:14268/api/traces. Only used if the tracing endpoint is unset." introductionVersion:"5.0"`
}
// Convert Tracing to the tracing package's Config struct.
func (t Tracing) Convert() tracing.Config {
return tracing.Config{
Enabled: t.Enabled,
Type: t.Type,
Endpoint: t.Endpoint,
Collector: t.Collector,
}
}

View File

@@ -0,0 +1,17 @@
package logging
import (
"github.com/owncloud/ocis/v2/ocis-pkg/log"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config"
)
// Configure initializes a service-specific logger instance.
func Configure(name string, cfg *config.Log) log.Logger {
return log.NewLogger(
log.Name(name),
log.Level(cfg.Level),
log.Pretty(cfg.Pretty),
log.Color(cfg.Color),
log.File(cfg.File),
)
}

View File

@@ -0,0 +1,35 @@
package metrics
import "github.com/prometheus/client_golang/prometheus"
var (
// Namespace defines the namespace for the defines metrics.
Namespace = "ocis"
// Subsystem defines the subsystem for the defines metrics.
Subsystem = "activitylog"
)
// Metrics defines the available metrics of this service.
type Metrics struct {
BuildInfo *prometheus.GaugeVec
}
// New initializes the available metrics.
func New() *Metrics {
m := &Metrics{
BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: Namespace,
Subsystem: Subsystem,
Name: "build_info",
Help: "Build information",
}, []string{"version"}),
}
_ = prometheus.Register(
m.BuildInfo,
)
// TODO: implement metrics
return m
}

View File

@@ -0,0 +1,74 @@
package service
import (
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config"
microstore "go-micro.dev/v4/store"
"go.opentelemetry.io/otel/trace"
)
// Option for the activitylog service
type Option func(*Options)
// Options for the activitylog service
type Options struct {
Logger log.Logger
Config *config.Config
TraceProvider trace.TracerProvider
Stream events.Stream
RegisteredEvents []events.Unmarshaller
Store microstore.Store
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
}
// Logger configures a logger for the activitylog service
func Logger(log log.Logger) Option {
return func(o *Options) {
o.Logger = log
}
}
// Config adds the config for the activitylog service
func Config(c *config.Config) Option {
return func(o *Options) {
o.Config = c
}
}
// TraceProvider adds a tracer provider for the activitylog service
func TraceProvider(tp trace.TracerProvider) Option {
return func(o *Options) {
o.TraceProvider = tp
}
}
// Stream configures an event stream for the clientlog service
func Stream(s events.Stream) Option {
return func(o *Options) {
o.Stream = s
}
}
// RegisteredEvents registers the events the service should listen to
func RegisteredEvents(e []events.Unmarshaller) Option {
return func(o *Options) {
o.RegisteredEvents = e
}
}
// Store configures the store to use
func Store(store microstore.Store) Option {
return func(o *Options) {
o.Store = store
}
}
// GatewaySelector adds a grpc client selector for the gateway service
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
return func(o *Options) {
o.GatewaySelector = gatewaySelector
}
}

View File

@@ -0,0 +1,255 @@
package service
import (
"encoding/json"
"errors"
"fmt"
"path/filepath"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
"github.com/owncloud/ocis/v2/services/activitylog/pkg/config"
microstore "go-micro.dev/v4/store"
)
// Activity represents an activity
type Activity struct {
EventID string `json:"event_id"`
Depth int `json:"depth"`
Timestamp time.Time `json:"timestamp"`
}
// ActivitylogService logs events per resource
type ActivitylogService struct {
cfg *config.Config
log log.Logger
events <-chan events.Event
store microstore.Store
gws pool.Selectable[gateway.GatewayAPIClient]
}
// New creates a new ActivitylogService
func New(opts ...Option) (*ActivitylogService, error) {
o := &Options{}
for _, opt := range opts {
opt(o)
}
if o.Stream == nil {
return nil, errors.New("stream is required")
}
if o.Store == nil {
return nil, errors.New("store is required")
}
ch, err := events.Consume(o.Stream, o.Config.Service.Name, o.RegisteredEvents...)
if err != nil {
return nil, err
}
s := &ActivitylogService{
log: o.Logger,
cfg: o.Config,
events: ch,
store: o.Store,
gws: o.GatewaySelector,
}
return s, nil
}
// Run runs the service
func (a *ActivitylogService) Run() error {
for e := range a.events {
var err error
switch ev := e.Event.(type) {
case events.UploadReady:
err = a.AddActivity(ev.FileRef, e.ID, utils.TSToTime(ev.Timestamp))
case events.FileTouched:
err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
case events.ContainerCreated:
err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemTrashed:
err = a.AddActivityTrashed(ev.ID, ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemPurged:
err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemMoved:
err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
case events.ShareCreated:
err = a.AddActivity(toRef(ev.ItemID), e.ID, utils.TSToTime(ev.CTime))
case events.ShareUpdated:
err = a.AddActivity(toRef(ev.ItemID), e.ID, utils.TSToTime(ev.MTime))
case events.ShareRemoved:
err = a.AddActivity(toRef(ev.ItemID), e.ID, ev.Timestamp)
case events.LinkCreated:
err = a.AddActivity(toRef(ev.ItemID), e.ID, utils.TSToTime(ev.CTime))
case events.LinkUpdated:
err = a.AddActivity(toRef(ev.ItemID), e.ID, utils.TSToTime(ev.CTime))
case events.LinkRemoved:
err = a.AddActivity(toRef(ev.ItemID), e.ID, utils.TSToTime(ev.Timestamp))
case events.SpaceShared:
err = a.AddActivity(sToRef(ev.ID), e.ID, ev.Timestamp)
case events.SpaceShareUpdated:
err = a.AddActivity(sToRef(ev.ID), e.ID, ev.Timestamp)
case events.SpaceUnshared:
err = a.AddActivity(sToRef(ev.ID), e.ID, ev.Timestamp)
}
if err != nil {
a.log.Error().Err(err).Interface("event", e).Msg("could not process event")
}
}
return nil
}
// AddActivity adds the activity to the given resource and all its parents
func (a *ActivitylogService) AddActivity(initRef *provider.Reference, eventID string, timestamp time.Time) error {
gwc, err := a.gws.Next()
if err != nil {
return fmt.Errorf("cant get gateway client: %w", err)
}
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("cant get service user context: %w", err)
}
return a.addActivity(initRef, eventID, timestamp, func(ref *provider.Reference) (*provider.ResourceInfo, error) {
return utils.GetResource(ctx, ref, gwc)
})
}
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
func (a *ActivitylogService) AddActivityTrashed(resourceID *provider.ResourceId, reference *provider.Reference, eventID string, timestamp time.Time) error {
gwc, err := a.gws.Next()
if err != nil {
return fmt.Errorf("cant get gateway client: %w", err)
}
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("cant get service user context: %w", err)
}
// store activity on trashed item
if err := a.storeActivity(resourceID, eventID, 0, timestamp); err != nil {
return fmt.Errorf("could not store activity: %w", err)
}
// get previous parent
ref := &provider.Reference{
ResourceId: reference.GetResourceId(),
Path: filepath.Dir(reference.GetPath()),
}
return a.addActivity(ref, eventID, timestamp, func(ref *provider.Reference) (*provider.ResourceInfo, error) {
return utils.GetResource(ctx, ref, gwc)
})
}
// Activities returns the activities for the given resource
func (a *ActivitylogService) Activities(rid *provider.ResourceId) ([]Activity, error) {
resourceID := storagespace.FormatResourceID(*rid)
records, err := a.store.Read(resourceID)
if err != nil && err != microstore.ErrNotFound {
return nil, fmt.Errorf("could not read activities: %w", err)
}
if len(records) == 0 {
return []Activity{}, nil
}
var activities []Activity
if err := json.Unmarshal(records[0].Value, &activities); err != nil {
return nil, fmt.Errorf("could not unmarshal activities: %w", err)
}
return activities, nil
}
// note: getResource is abstracted to allow unit testing, in general this will just be utils.GetResource
func (a *ActivitylogService) addActivity(initRef *provider.Reference, eventID string, timestamp time.Time, getResource func(*provider.Reference) (*provider.ResourceInfo, error)) error {
var (
info *provider.ResourceInfo
err error
depth int
ref = initRef
)
for {
info, err = getResource(ref)
if err != nil {
return fmt.Errorf("could not get resource info: %w", err)
}
if err := a.storeActivity(info.GetId(), eventID, depth, timestamp); err != nil {
return fmt.Errorf("could not store activity: %w", err)
}
if info != nil && utils.IsSpaceRoot(info) {
return nil
}
depth++
ref = &provider.Reference{ResourceId: info.GetParentId()}
}
}
func (a *ActivitylogService) storeActivity(rid *provider.ResourceId, eventID string, depth int, timestamp time.Time) error {
if rid == nil {
return errors.New("resource id is required")
}
resourceID := storagespace.FormatResourceID(*rid)
records, err := a.store.Read(resourceID)
if err != nil && err != microstore.ErrNotFound {
return err
}
var activities []Activity
if len(records) > 0 {
if err := json.Unmarshal(records[0].Value, &activities); err != nil {
return err
}
}
// TODO: max len check?
activities = append(activities, Activity{
EventID: eventID,
Depth: depth,
Timestamp: timestamp,
})
b, err := json.Marshal(activities)
if err != nil {
return err
}
return a.store.Write(&microstore.Record{
Key: resourceID,
Value: b,
})
}
func toRef(r *provider.ResourceId) *provider.Reference {
return &provider.Reference{
ResourceId: r,
}
}
func sToRef(s *provider.StorageSpaceId) *provider.Reference {
return &provider.Reference{
ResourceId: &provider.ResourceId{
OpaqueId: s.GetOpaqueId(),
SpaceId: s.GetOpaqueId(),
},
}
}

View File

@@ -0,0 +1,181 @@
package service
import (
"testing"
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/store"
"github.com/stretchr/testify/require"
)
func TestAddActivity(t *testing.T) {
testCases := []struct {
Name string
Tree map[string]*provider.ResourceInfo
Activities map[string]string
Expected map[string][]Activity
}{
{
Name: "simple",
Tree: map[string]*provider.ResourceInfo{
"base": resourceInfo("base", "parent"),
"parent": resourceInfo("parent", "spaceid"),
"spaceid": resourceInfo("spaceid", "spaceid"),
},
Activities: map[string]string{
"activity": "base",
},
Expected: map[string][]Activity{
"base": activitites("activity", 0),
"parent": activitites("activity", 1),
"spaceid": activitites("activity", 2),
},
},
{
Name: "two activities on same resource",
Tree: map[string]*provider.ResourceInfo{
"base": resourceInfo("base", "parent"),
"parent": resourceInfo("parent", "spaceid"),
"spaceid": resourceInfo("spaceid", "spaceid"),
},
Activities: map[string]string{
"activity1": "base",
"activity2": "base",
},
Expected: map[string][]Activity{
"base": activitites("activity1", 0, "activity2", 0),
"parent": activitites("activity1", 1, "activity2", 1),
"spaceid": activitites("activity1", 2, "activity2", 2),
},
},
{
Name: "two activities on different resources",
Tree: map[string]*provider.ResourceInfo{
"base1": resourceInfo("base1", "parent"),
"base2": resourceInfo("base2", "parent"),
"parent": resourceInfo("parent", "spaceid"),
"spaceid": resourceInfo("spaceid", "spaceid"),
},
Activities: map[string]string{
"activity1": "base1",
"activity2": "base2",
},
Expected: map[string][]Activity{
"base1": activitites("activity1", 0),
"base2": activitites("activity2", 0),
"parent": activitites("activity1", 1, "activity2", 1),
"spaceid": activitites("activity1", 2, "activity2", 2),
},
},
{
Name: "more elaborate resource tree",
Tree: map[string]*provider.ResourceInfo{
"base1": resourceInfo("base1", "parent1"),
"base2": resourceInfo("base2", "parent1"),
"parent1": resourceInfo("parent1", "spaceid"),
"base3": resourceInfo("base3", "parent2"),
"parent2": resourceInfo("parent2", "spaceid"),
"spaceid": resourceInfo("spaceid", "spaceid"),
},
Activities: map[string]string{
"activity1": "base1",
"activity2": "base2",
"activity3": "base3",
},
Expected: map[string][]Activity{
"base1": activitites("activity1", 0),
"base2": activitites("activity2", 0),
"base3": activitites("activity3", 0),
"parent1": activitites("activity1", 1, "activity2", 1),
"parent2": activitites("activity3", 1),
"spaceid": activitites("activity1", 2, "activity2", 2, "activity3", 2),
},
},
{
Name: "different depths within one resource",
Tree: map[string]*provider.ResourceInfo{
"base1": resourceInfo("base1", "parent1"),
"parent1": resourceInfo("parent1", "parent2"),
"base2": resourceInfo("base2", "parent2"),
"parent2": resourceInfo("parent2", "parent3"),
"base3": resourceInfo("base3", "parent3"),
"parent3": resourceInfo("parent3", "spaceid"),
"spaceid": resourceInfo("spaceid", "spaceid"),
},
Activities: map[string]string{
"activity1": "base1",
"activity2": "base2",
"activity3": "base3",
"activity4": "parent2",
},
Expected: map[string][]Activity{
"base1": activitites("activity1", 0),
"base2": activitites("activity2", 0),
"base3": activitites("activity3", 0),
"parent1": activitites("activity1", 1),
"parent2": activitites("activity1", 2, "activity2", 1, "activity4", 0),
"parent3": activitites("activity1", 3, "activity2", 2, "activity3", 1, "activity4", 1),
"spaceid": activitites("activity1", 4, "activity2", 3, "activity3", 2, "activity4", 2),
},
},
}
for _, tc := range testCases {
alog := &ActivitylogService{
store: store.Create(),
}
getResource := func(ref *provider.Reference) (*provider.ResourceInfo, error) {
return tc.Tree[ref.GetResourceId().GetOpaqueId()], nil
}
for k, v := range tc.Activities {
err := alog.addActivity(reference(v), k, time.Time{}, getResource)
require.NoError(t, err)
}
for id, acts := range tc.Expected {
activities, err := alog.Activities(resourceID(id))
require.NoError(t, err, tc.Name+":"+id)
require.ElementsMatch(t, acts, activities, tc.Name+":"+id)
}
}
}
func activitites(acts ...interface{}) []Activity {
var activities []Activity
act := Activity{}
for _, a := range acts {
switch v := a.(type) {
case string:
act.EventID = v
case int:
act.Depth = v
activities = append(activities, act)
}
}
return activities
}
func resourceID(id string) *provider.ResourceId {
return &provider.ResourceId{
StorageId: "storageid",
OpaqueId: id,
SpaceId: "spaceid",
}
}
func reference(id string) *provider.Reference {
return &provider.Reference{ResourceId: resourceID(id)}
}
func resourceInfo(id, parentID string) *provider.ResourceInfo {
return &provider.ResourceInfo{
Id: resourceID(id),
ParentId: resourceID(parentID),
Space: &provider.StorageSpace{
Root: resourceID("spaceid"),
},
}
}