Merge pull request #3288 from kobergj/BringAuditToMonorepo

Bring audit to monorepo
This commit is contained in:
Willy Kloucek
2022-03-10 09:37:59 +01:00
committed by GitHub
22 changed files with 644 additions and 0 deletions

View File

@@ -0,0 +1,14 @@
package main
import (
"os"
"github.com/owncloud/ocis/audit/pkg/command"
"github.com/owncloud/ocis/audit/pkg/config"
)
func main() {
if err := command.Execute(config.DefaultConfig()); err != nil {
os.Exit(1)
}
}

View File

@@ -0,0 +1,18 @@
package command
import (
"github.com/owncloud/ocis/audit/pkg/config"
"github.com/urfave/cli/v2"
)
// Health is the entrypoint for the health command.
func Health(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "health",
Usage: "Check health status",
Action: func(c *cli.Context) error {
// Not implemented
return nil
},
}
}

64
audit/pkg/command/root.go Normal file
View File

@@ -0,0 +1,64 @@
package command
import (
"context"
"os"
"github.com/owncloud/ocis/audit/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/clihelper"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// GetCommands provides all commands for this service
func GetCommands(cfg *config.Config) cli.Commands {
return []*cli.Command{
// start this service
Server(cfg),
// interaction with this service
// infos about this service
Health(cfg),
Version(cfg),
}
}
// Execute is the entry point for the audit command.
func Execute(cfg *config.Config) error {
app := clihelper.DefaultApp(&cli.App{
Name: "audit",
Usage: "starts audit service",
Commands: GetCommands(cfg),
})
cli.HelpFlag = &cli.BoolFlag{
Name: "help,h",
Usage: "Show the help",
}
return app.Run(os.Args)
}
// SutureService allows for the audit command to be embedded and supervised by a suture supervisor tree.
type SutureService struct {
cfg *config.Config
}
// NewSutureService creates a new audit.SutureService
func NewSutureService(cfg *ociscfg.Config) suture.Service {
cfg.Settings.Commons = cfg.Commons
return SutureService{
cfg: cfg.Audit,
}
}
func (s SutureService) Serve(ctx context.Context) error {
s.cfg.Context = ctx
if err := Execute(s.cfg); err != nil {
return err
}
return nil
}

View File

@@ -0,0 +1,51 @@
package command
import (
"context"
"fmt"
"github.com/asim/go-micro/plugins/events/nats/v4"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/events/server"
"github.com/owncloud/ocis/audit/pkg/config"
"github.com/owncloud/ocis/audit/pkg/config/parser"
"github.com/owncloud/ocis/audit/pkg/logging"
svc "github.com/owncloud/ocis/audit/pkg/service"
"github.com/owncloud/ocis/audit/pkg/types"
"github.com/urfave/cli/v2"
)
// Server is the entrypoint for the server command.
func Server(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "server",
Usage: fmt.Sprintf("start %s extension without runtime (unsupervised mode)", cfg.Service.Name),
Category: "server",
Before: func(c *cli.Context) error {
return parser.ParseConfig(cfg)
},
Action: func(c *cli.Context) error {
logger := logging.Configure(cfg.Service.Name, cfg.Log)
ctx := cfg.Context
if ctx == nil {
ctx = context.Background()
}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
evtsCfg := cfg.Events
client, err := server.NewNatsStream(nats.Address(evtsCfg.Endpoint), nats.ClusterID(evtsCfg.Cluster))
if err != nil {
return err
}
evts, err := events.Consume(client, evtsCfg.ConsumerGroup, types.RegisteredEvents()...)
if err != nil {
return err
}
svc.AuditLoggerFromConfig(ctx, cfg.Auditlog, evts, logger)
return nil
},
}
}

View File

@@ -0,0 +1,19 @@
package command
import (
"github.com/owncloud/ocis/audit/pkg/config"
"github.com/urfave/cli/v2"
)
// Version prints the service versions of all running instances.
func Version(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "version",
Usage: "print the version of this binary and the running extension instances",
Category: "info",
Action: func(c *cli.Context) error {
// not implemented
return nil
},
}
}

View File

@@ -0,0 +1,37 @@
package config
import (
"context"
"github.com/owncloud/ocis/ocis-pkg/shared"
)
// Config combines all available configuration parts.
type Config struct {
*shared.Commons
Service Service
Log *Log `ocisConfig:"log"`
Debug Debug `ocisConfig:"debug"`
Events Events `ocisConfig:"events"`
Auditlog Auditlog `ocisConfig:"auditlog"`
Context context.Context
}
// Events combines the configuration options for the event bus.
type Events struct {
Endpoint string `ocisConfig:"events_endpoint" env:"AUDIT_EVENTS_ENDPOINT" desc:"the address of the streaming service"`
Cluster string `ocisConfig:"events_cluster" env:"AUDIT_EVENTS_CLUSTER" desc:"the clusterID of the streaming service. Mandatory when using nats"`
ConsumerGroup string `ocisConfig:"events_group" env:"AUDIT_EVENTS_GROUP" desc:"the customergroup of the service. One group will only get one vopy of an event"`
}
// Auditlog holds audit log information
type Auditlog struct {
LogToConsole bool `ocisConfig:"log_to_console" env:"AUDIT_LOG_TO_CONSOLE" desc:"logs to Stdout if true"`
LogToFile bool `ocisConfig:"log_to_file" env:"AUDIT_LOG_TO_FILE" desc:"logs to file if true"`
FilePath string `ocisConfig:"filepath" env:"AUDIT_FILEPATH" desc:"filepath to the logfile. Mandatory if LogToFile is true"`
Format string `ocisConfig:"format" env:"AUDIT_FORMAT" desc:"log format. using json is advised"`
}

View File

@@ -0,0 +1,9 @@
package config
// Debug defines the available debug configuration.
type Debug struct {
Addr string `ocisConfig:"addr" env:"AUDIT_DEBUG_ADDR"`
Token string `ocisConfig:"token" env:"AUDIT_DEBUG_TOKEN"`
Pprof bool `ocisConfig:"pprof" env:"AUDIT_DEBUG_PPROF"`
Zpages bool `ocisConfig:"zpages" env:"AUDIT_DEBUG_ZPAGES"`
}

View File

@@ -0,0 +1,18 @@
package config
func DefaultConfig() *Config {
return &Config{
Service: Service{
Name: "audit",
},
Events: Events{
Endpoint: "127.0.0.1:9233",
Cluster: "test-cluster",
ConsumerGroup: "audit",
},
Auditlog: Auditlog{
LogToConsole: true,
Format: "json",
},
}
}

9
audit/pkg/config/log.go Normal file
View File

@@ -0,0 +1,9 @@
package config
// Log defines the available log configuration.
type Log struct {
Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;AUDIT_LOG_LEVEL"`
Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;AUDIT_LOG_PRETTY"`
Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;AUDIT_LOG_COLOR"`
File string `mapstructure:"file" env:"OCIS_LOG_FILE;AUDIT_LOG_FILE"`
}

View File

@@ -0,0 +1,40 @@
package parser
import (
"errors"
"github.com/owncloud/ocis/audit/pkg/config"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/config/envdecode"
)
// ParseConfig loads accounts configuration from known paths.
func ParseConfig(cfg *config.Config) error {
_, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg)
if err != nil {
return err
}
// provide with defaults for shared logging, since we need a valid destination address for BindEnv.
if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
cfg.Log = &config.Log{
Level: cfg.Commons.Log.Level,
Pretty: cfg.Commons.Log.Pretty,
Color: cfg.Commons.Log.Color,
File: cfg.Commons.Log.File,
}
} else if cfg.Log == nil {
cfg.Log = &config.Log{}
}
// load all env variables relevant to the config in the current context.
if err := envdecode.Decode(cfg); err != nil {
// no environment variable set for this config is an expected "error"
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
return err
}
}
return nil
}

View File

@@ -0,0 +1,6 @@
package config
// Service defines the available service configuration.
type Service struct {
Name string
}

View File

@@ -0,0 +1,17 @@
package logging
import (
"github.com/owncloud/ocis/audit/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
)
// LoggerFromConfig initializes a service-specific logger instance.
func Configure(name string, cfg *config.Log) log.Logger {
return log.NewLogger(
log.Name(name),
log.Level(cfg.Level),
log.Pretty(cfg.Pretty),
log.Color(cfg.Color),
log.File(cfg.File),
)
}

View File

@@ -0,0 +1,99 @@
package svc
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/owncloud/ocis/audit/pkg/config"
"github.com/owncloud/ocis/audit/pkg/types"
"github.com/owncloud/ocis/ocis-pkg/log"
)
// Log is used to log to different outputs
type Log func([]byte)
// Marshaller is used to marshal events
type Marshaller func(interface{}) ([]byte, error)
// AuditLoggerFromConfig will start a new AuditLogger generated from the config
func AuditLoggerFromConfig(ctx context.Context, cfg config.Auditlog, ch <-chan interface{}, log log.Logger) {
var logs []Log
if cfg.LogToConsole {
logs = append(logs, WriteToStdout())
}
if cfg.LogToFile {
logs = append(logs, WriteToFile(cfg.FilePath, log))
}
StartAuditLogger(ctx, ch, log, Marshal(cfg.Format, log), logs...)
}
// StartAuditLogger will block. run in seperate go routine
func StartAuditLogger(ctx context.Context, ch <-chan interface{}, log log.Logger, marshaller Marshaller, logto ...Log) {
for {
select {
case <-ctx.Done():
return
case i := <-ch:
var auditEvent interface{}
switch ev := i.(type) {
case events.ShareCreated:
auditEvent = types.ShareCreated(ev)
default:
log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev))
continue
}
b, err := marshaller(auditEvent)
if err != nil {
log.Error().Err(err).Msg("error marshaling the event")
continue
}
for _, l := range logto {
l(b)
}
}
}
}
// WriteToFile returns a Log function writing to a file
func WriteToFile(path string, log log.Logger) Log {
return func(content []byte) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
log.Error().Err(err).Msgf("error opening file '%s'", path)
return
}
defer file.Close()
if _, err := fmt.Fprintln(file, string(content)); err != nil {
log.Error().Err(err).Msgf("error writing to file '%s'", path)
}
}
}
// WriteToStdout return a Log function writing to Stdout
func WriteToStdout() Log {
return func(content []byte) {
fmt.Println(string(content))
}
}
// Marshal returns a Marshaller from the `format` string
func Marshal(format string, log log.Logger) Marshaller {
switch format {
default:
log.Error().Msgf("unknown format '%s'", format)
return nil
case "json":
return json.Marshal
}
}

View File

@@ -0,0 +1,80 @@
package svc
import (
"context"
"encoding/json"
"testing"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/owncloud/ocis/audit/pkg/types"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/test-go/testify/require"
group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
rtypes "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
)
var testCases = []struct {
Alias string
SystemEvent interface{}
CheckAuditEvent func(*testing.T, []byte)
}{
{
Alias: "ShareCreated",
SystemEvent: events.ShareCreated{
Sharer: &user.UserId{
OpaqueId: "sharing-userid",
Idp: "idp",
},
GranteeUserID: &user.UserId{
OpaqueId: "beshared-userid",
Idp: "idp",
},
GranteeGroupID: &group.GroupId{},
Sharee: &provider.Grantee{},
ItemID: &provider.ResourceId{
StorageId: "storage-1",
OpaqueId: "itemid-1",
},
CTime: &rtypes.Timestamp{
Seconds: 0,
Nanos: 0,
},
},
CheckAuditEvent: func(t *testing.T, b []byte) {
ev := types.AuditEventShareCreated{}
err := json.Unmarshal(b, &ev)
require.NoError(t, err)
require.Equal(t, ev.User, "sharing-userid")
require.Equal(t, ev.ShareWith, "beshared-userid")
require.Equal(t, ev.FileID, "itemid-1")
require.Equal(t, ev.Time, "1970-01-01T01:00:00+01:00")
},
},
}
func TestAuditLogging(t *testing.T) {
log := log.NewLogger()
inch := make(chan interface{})
defer close(inch)
outch := make(chan []byte)
defer close(outch)
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
go StartAuditLogger(ctx, inch, log, Marshal("json", log), func(b []byte) {
outch <- b
})
for _, tc := range testCases {
inch <- tc.SystemEvent
tc.CheckAuditEvent(t, <-outch)
}
}

View File

@@ -0,0 +1,79 @@
package types
import (
"fmt"
"time"
"github.com/cs3org/reva/v2/pkg/events"
)
// actions
const (
actionShareCreated = "file_shared"
)
// messages
const (
messageShareCreated = "user '%s' shared file '%s' with '%s'"
)
// BasicAuditEvent creates an AuditEvent from given values
func BasicAuditEvent(uid string, ctime string, msg string, action string) AuditEvent {
return AuditEvent{
User: uid,
Time: ctime,
App: "admin_audit",
Message: msg,
Action: action,
Level: 1,
// NOTE: those values are not in the events and can therefore not be filled at the moment
RemoteAddr: "",
URL: "",
Method: "",
UserAgent: "",
CLI: false,
}
}
// SharingAuditEvent creates an AuditEventSharing from given values
func SharingAuditEvent(fileid string, uid string, base AuditEvent) AuditEventSharing {
return AuditEventSharing{
AuditEvent: base,
FileID: fileid,
Owner: uid,
// NOTE: those values are not in the events and can therefore not be filled at the moment
ShareID: "",
Path: "",
}
}
// ShareCreated converts a ShareCreated Event to an AuditEventShareCreated
func ShareCreated(ev events.ShareCreated) AuditEventShareCreated {
with := ""
typ := ""
if ev.GranteeUserID != nil && ev.GranteeUserID.OpaqueId != "" {
with = ev.GranteeUserID.OpaqueId
typ = "user"
} else if ev.GranteeGroupID != nil && ev.GranteeGroupID.OpaqueId != "" {
with = ev.GranteeGroupID.OpaqueId
typ = "group"
}
uid := ev.Sharer.OpaqueId
t := time.Unix(int64(ev.CTime.Seconds), int64(ev.CTime.Nanos)).Format(time.RFC3339)
base := BasicAuditEvent(uid, t, fmt.Sprintf(messageShareCreated, uid, ev.ItemID.OpaqueId, with), actionShareCreated)
return AuditEventShareCreated{
AuditEventSharing: SharingAuditEvent(ev.ItemID.OpaqueId, uid, base),
ShareOwner: uid,
ShareWith: with,
ShareType: typ,
// NOTE: those values are not in the events and can therefore not be filled at the moment
ItemType: "",
ExpirationDate: "",
SharePass: false,
Permissions: "",
ShareToken: "",
}
}

12
audit/pkg/types/events.go Normal file
View File

@@ -0,0 +1,12 @@
package types
import (
"github.com/cs3org/reva/v2/pkg/events"
)
// RegisteredEvents returns the events the service is registered for
func RegisteredEvents() []events.Unmarshaller {
return []events.Unmarshaller{
events.ShareCreated{},
}
}

40
audit/pkg/types/types.go Normal file
View File

@@ -0,0 +1,40 @@
package types
// AuditEvent is the basic audit event
type AuditEvent struct {
RemoteAddr string // the remote client IP
User string // the UID of the user performing the action. Or "IP x.x.x.x.", "cron", "CLI", "unknown"
URL string // the process request URI
Method string // the HTTP request method
UserAgent string // the HTTP request user agent
Time string // the time of the event eg: 2018-05-08T08:26:00+00:00
App string // always 'admin_audit'
Message string // sentence explaining the action
Action string // unique action identifier eg: file_delete or public_link_created
CLI bool // if the action was performed from the CLI
Level int // the log level of the entry (usually 1 for audit events)
}
// AuditEventSharing is the basic audit event for shares
type AuditEventSharing struct {
AuditEvent
FileID string // The file identifier for the item shared.
Owner string // The UID of the owner of the shared item.
Path string // The path to the shared item.
ShareID string // The sharing identifier. (not available for public_link_accessed or when recipient unshares)
}
// AuditEventShareCreated is the event logged when a share is created
type AuditEventShareCreated struct {
AuditEventSharing
ItemType string // file or folder
ExpirationDate string // The text expiration date in format 'yyyy-mm-dd'
SharePass bool // If the share is password protected.
Permissions string // The permissions string eg: "READ"
ShareType string // group user or link
ShareWith string // The UID or GID of the share recipient. (not available for public link)
ShareOwner string // The UID of the share owner.
ShareToken string // For link shares the unique token, else null
}

1
go.mod
View File

@@ -61,6 +61,7 @@ require (
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.3.0
github.com/stretchr/testify v1.7.0
github.com/test-go/testify v1.1.4
github.com/thejerf/suture/v4 v4.0.2
github.com/urfave/cli/v2 v2.3.0
go-micro.dev/v4 v4.6.0

View File

@@ -4,6 +4,7 @@ import (
"github.com/owncloud/ocis/ocis-pkg/shared"
accounts "github.com/owncloud/ocis/accounts/pkg/config"
audit "github.com/owncloud/ocis/audit/pkg/config"
glauth "github.com/owncloud/ocis/glauth/pkg/config"
graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/config"
graph "github.com/owncloud/ocis/graph/pkg/config"
@@ -58,6 +59,7 @@ type Config struct {
TokenManager TokenManager `ocisConfig:"token_manager"`
Runtime Runtime `ocisConfig:"runtime"`
Audit *audit.Config `ocsiConfig:"audit"`
Accounts *accounts.Config `ocisConfig:"accounts"`
GLAuth *glauth.Config `ocisConfig:"glauth"`
Graph *graph.Config `ocisConfig:"graph"`

View File

@@ -2,6 +2,7 @@ package config
import (
accounts "github.com/owncloud/ocis/accounts/pkg/config"
audit "github.com/owncloud/ocis/audit/pkg/config"
glauth "github.com/owncloud/ocis/glauth/pkg/config"
graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/config"
graph "github.com/owncloud/ocis/graph/pkg/config"
@@ -28,6 +29,7 @@ func DefaultConfig() *Config {
Port: "9250",
Host: "localhost",
},
Audit: audit.DefaultConfig(),
Accounts: accounts.DefaultConfig(),
GLAuth: glauth.DefaultConfig(),
Graph: graph.DefaultConfig(),

26
ocis/pkg/command/audit.go Normal file
View File

@@ -0,0 +1,26 @@
package command
import (
"github.com/owncloud/ocis/audit/pkg/command"
"github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/config/parser"
"github.com/owncloud/ocis/ocis/pkg/register"
"github.com/urfave/cli/v2"
)
// AuditCommand is the entrypoint for the audit command.
func AuditCommand(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "audit",
Usage: "start audit service",
Category: "extensions",
Before: func(ctx *cli.Context) error {
return parser.ParseConfig(cfg)
},
Subcommands: command.GetCommands(cfg.Audit),
}
}
func init() {
register.AddCommand(AuditCommand)
}

View File

@@ -18,6 +18,7 @@ import (
mzlog "github.com/asim/go-micro/plugins/logger/zerolog/v4"
"github.com/mohae/deepcopy"
"github.com/olekukonko/tablewriter"
accounts "github.com/owncloud/ocis/accounts/pkg/command"
glauth "github.com/owncloud/ocis/glauth/pkg/command"
graphExplorer "github.com/owncloud/ocis/graph-explorer/pkg/command"