Migrate frontend to new service tracing setup.

This alters the frontend to use the new service tracing setup.
This commit is contained in:
Daniël Franke
2023-06-29 10:40:00 +02:00
parent f15ed6bc91
commit 7a229e438b
21 changed files with 98 additions and 83 deletions

2
go.mod
View File

@@ -13,7 +13,7 @@ require (
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/coreos/go-oidc/v3 v3.6.0
github.com/cs3org/go-cs3apis v0.0.0-20230516150832-730ac860c71d
github.com/cs3org/reva/v2 v2.14.1-0.20230623085734-919a9585f147
github.com/cs3org/reva/v2 v2.14.1-0.20230629081848-5e7f1bf5c21d
github.com/disintegration/imaging v1.6.2
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
github.com/egirna/icap-client v0.1.1

2
go.sum
View File

@@ -624,6 +624,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo
github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4=
github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc=
github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA=
github.com/cs3org/reva/v2 v2.14.1-0.20230629081848-5e7f1bf5c21d h1:D/B1j72MC/IP0DIMOJdmt7rCBKVps2Xob1MSLGHZQ1A=
github.com/cs3org/reva/v2 v2.14.1-0.20230629081848-5e7f1bf5c21d/go.mod h1:E32krZG159YflDSjDWfx/QGIC2529PS5LiPnGNHu3d0=
github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=

View File

@@ -12,13 +12,13 @@ import (
"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
"github.com/owncloud/ocis/v2/ocis-pkg/registry"
"github.com/owncloud/ocis/v2/ocis-pkg/sync"
"github.com/owncloud/ocis/v2/ocis-pkg/tracing"
"github.com/owncloud/ocis/v2/ocis-pkg/version"
"github.com/owncloud/ocis/v2/services/frontend/pkg/config"
"github.com/owncloud/ocis/v2/services/frontend/pkg/config/parser"
"github.com/owncloud/ocis/v2/services/frontend/pkg/logging"
"github.com/owncloud/ocis/v2/services/frontend/pkg/revaconfig"
"github.com/owncloud/ocis/v2/services/frontend/pkg/server/debug"
"github.com/owncloud/ocis/v2/services/frontend/pkg/tracing"
"github.com/urfave/cli/v2"
)
@@ -33,7 +33,7 @@ func Server(cfg *config.Config) *cli.Command {
},
Action: func(c *cli.Context) error {
logger := logging.Configure(cfg.Service.Name, cfg.Log)
err := tracing.Configure(cfg, logger)
tracingProvider, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name)
if err != nil {
return err
}
@@ -54,6 +54,7 @@ func Server(cfg *config.Config) *cli.Command {
runtime.RunWithOptions(rCfg, pidFile,
runtime.WithLogger(&logger.Logger),
runtime.WithRegistry(reg),
runtime.WithTraceProvider(tracingProvider),
)
return nil
@@ -71,7 +72,6 @@ func Server(cfg *config.Config) *cli.Command {
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", "debug").Msg("Failed to initialize server")
return err

View File

@@ -55,12 +55,6 @@ type Config struct {
Supervised bool `yaml:"-"`
Context context.Context `yaml:"-"`
}
type Tracing struct {
Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;FRONTEND_TRACING_ENABLED" desc:"Activates tracing."`
Type string `yaml:"type" env:"OCIS_TRACING_TYPE;FRONTEND_TRACING_TYPE" desc:"The type of tracing. Defaults to \"\", which is the same as \"jaeger\". Allowed tracing types are \"jaeger\" and \"\" as of now."`
Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;FRONTEND_TRACING_ENDPOINT" desc:"The endpoint of the tracing agent."`
Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;FRONTEND_TRACING_COLLECTOR" desc:"The HTTP endpoint for sending spans directly to a collector, i.e. http://jaeger-collector:14268/api/traces. Only used if the tracing endpoint is unset."`
}
type Log struct {
Level string `yaml:"level" env:"OCIS_LOG_LEVEL;FRONTEND_LOG_LEVEL" desc:"The log level. Valid values are: \"panic\", \"fatal\", \"error\", \"warn\", \"info\", \"debug\", \"trace\"."`

View File

@@ -0,0 +1,20 @@
package config
import "github.com/owncloud/ocis/v2/ocis-pkg/tracing"
type Tracing struct {
Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;FRONTEND_TRACING_ENABLED" desc:"Activates tracing."`
Type string `yaml:"type" env:"OCIS_TRACING_TYPE;FRONTEND_TRACING_TYPE" desc:"The type of tracing. Defaults to \"\", which is the same as \"jaeger\". Allowed tracing types are \"jaeger\" and \"\" as of now."`
Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;FRONTEND_TRACING_ENDPOINT" desc:"The endpoint of the tracing agent."`
Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;FRONTEND_TRACING_COLLECTOR" desc:"The HTTP endpoint for sending spans directly to a collector, i.e. http://jaeger-collector:14268/api/traces. Only used if the tracing endpoint is unset."`
}
// Convert Tracing to the tracing package's Config struct.
func (t Tracing) Convert() tracing.Config {
return tracing.Config{
Enabled: t.Enabled,
Type: t.Type,
Endpoint: t.Endpoint,
Collector: t.Collector,
}
}

View File

@@ -12,7 +12,6 @@ import (
// FrontendConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func FrontendConfigFromStruct(cfg *config.Config) (map[string]interface{}, error) {
webURL, err := url.Parse(cfg.PublicURL)
if err != nil {
return nil, err
@@ -71,13 +70,6 @@ func FrontendConfigFromStruct(cfg *config.Config) (map[string]interface{}, error
}
return map[string]interface{}{
"core": map[string]interface{}{
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_exporter": cfg.Tracing.Type,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": cfg.Service.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.TokenManager.JWTSecret,
"gatewaysvc": cfg.Reva.Address, // Todo or address?

View File

@@ -1,18 +0,0 @@
package tracing
import (
"github.com/owncloud/ocis/v2/ocis-pkg/log"
"github.com/owncloud/ocis/v2/ocis-pkg/tracing"
"github.com/owncloud/ocis/v2/services/frontend/pkg/config"
"go.opentelemetry.io/otel/trace"
)
var (
// TraceProvider is the global trace provider for the service.
TraceProvider = trace.NewNoopTracerProvider()
)
func Configure(cfg *config.Config, logger log.Logger) error {
tracing.Configure(cfg.Tracing.Enabled, cfg.Tracing.Type, logger)
return nil
}

View File

@@ -21,6 +21,7 @@ package runtime
import (
"github.com/rs/zerolog"
"go-micro.dev/v4/registry"
"go.opentelemetry.io/otel/trace"
)
// Option defines a single option function.
@@ -28,8 +29,9 @@ type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Logger *zerolog.Logger
Registry registry.Registry
Logger *zerolog.Logger
Registry registry.Registry
TraceProvider trace.TracerProvider
}
// newOptions initializes the available default options.
@@ -56,3 +58,10 @@ func WithRegistry(r registry.Registry) Option {
o.Registry = r
}
}
// WithTraceProvider provides a function to set the trace provider.
func WithTraceProvider(tp trace.TracerProvider) Option {
return func(o *Options) {
o.TraceProvider = tp
}
}

View File

@@ -58,7 +58,7 @@ func RunWithOptions(mainConf map[string]interface{}, pidFile string, opts ...Opt
panic(err)
}
run(mainConf, coreConf, options.Logger, pidFile)
run(mainConf, coreConf, options.Logger, options.TraceProvider, pidFile)
}
type coreConf struct {
@@ -74,11 +74,21 @@ type coreConf struct {
TracingService string `mapstructure:"tracing_service"`
}
func run(mainConf map[string]interface{}, coreConf *coreConf, logger *zerolog.Logger, filename string) {
func run(
mainConf map[string]interface{},
coreConf *coreConf,
logger *zerolog.Logger,
tp trace.TracerProvider,
filename string,
) {
host, _ := os.Hostname()
logger.Info().Msgf("host info: %s", host)
tp := initTracing(coreConf)
// Only initialise tracing if we didn't get a tracer provider.
if tp == nil {
logger.Debug().Msg("No pre-existing tracer given, initializing tracing")
tp = initTracing(coreConf)
}
initCPUCount(coreConf, logger)
servers := initServers(mainConf, logger, tp)
@@ -241,7 +251,7 @@ func getWriter(out string) (io.Writer, error) {
return os.Stdout, nil
}
fd, err := os.OpenFile(out, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
fd, err := os.OpenFile(out, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
err = errors.Wrap(err, "error creating log file: "+out)
return nil, err

View File

@@ -662,13 +662,15 @@ func (s *service) CreateContainer(ctx context.Context, req *provider.CreateConta
func (s *service) TouchFile(ctx context.Context, req *provider.TouchFileRequest) (*provider.TouchFileResponse, error) {
// FIXME these should be part of the TouchFileRequest object
var mtime string
if req.Opaque != nil {
if e, ok := req.Opaque.Map["lockid"]; ok && e.Decoder == "plain" {
ctx = ctxpkg.ContextSetLockID(ctx, string(e.Value))
}
mtime = utils.ReadPlainFromOpaque(req.Opaque, "X-OC-Mtime")
}
err := s.storage.TouchFile(ctx, req.Ref, utils.ExistsInOpaque(req.Opaque, "markprocessing"))
err := s.storage.TouchFile(ctx, req.Ref, utils.ExistsInOpaque(req.Opaque, "markprocessing"), mtime)
return &provider.TouchFileResponse{
Status: status.NewStatusFromErrType(ctx, "touch file", err),

View File

@@ -154,9 +154,17 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ
w.WriteHeader(http.StatusInternalServerError)
return
}
opaque := &typespb.Opaque{}
if mtime := r.Header.Get(net.HeaderOCMtime); mtime != "" {
utils.AppendPlainToOpaque(opaque, net.HeaderOCMtime, mtime)
// TODO: find a way to check if the storage really accepted the value
w.Header().Set(net.HeaderOCMtime, "accepted")
}
if length == 0 {
tfRes, err := client.TouchFile(ctx, &provider.TouchFileRequest{
Ref: ref,
Opaque: opaque,
Ref: ref,
})
if err != nil {
log.Error().Err(err).Msg("error sending grpc touch file request")
@@ -191,22 +199,7 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ
return
}
opaqueMap := map[string]*typespb.OpaqueEntry{
net.HeaderUploadLength: {
Decoder: "plain",
Value: []byte(strconv.FormatInt(length, 10)),
},
}
if mtime := r.Header.Get(net.HeaderOCMtime); mtime != "" {
opaqueMap[net.HeaderOCMtime] = &typespb.OpaqueEntry{
Decoder: "plain",
Value: []byte(mtime),
}
// TODO: find a way to check if the storage really accepted the value
w.Header().Set(net.HeaderOCMtime, "accepted")
}
utils.AppendPlainToOpaque(opaque, net.HeaderUploadLength, strconv.FormatInt(length, 10))
// curl -X PUT https://demo.owncloud.com/remote.php/webdav/testcs.bin -u demo:demo -d '123' -v -H 'OC-Checksum: SHA1:40bd001563085fc35165329ea1ff5c5ecbdbbeef'
@@ -231,16 +224,13 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ
// we do not check the algorithm here, because it might depend on the storage
if len(cparts) == 2 {
// Translate into TUS style Upload-Checksum header
opaqueMap[net.HeaderUploadChecksum] = &typespb.OpaqueEntry{
Decoder: "plain",
// algorithm is always lowercase, checksum is separated by space
Value: []byte(strings.ToLower(cparts[0]) + " " + cparts[1]),
}
// algorithm is always lowercase, checksum is separated by space
utils.AppendPlainToOpaque(opaque, net.HeaderUploadChecksum, strings.ToLower(cparts[0])+" "+cparts[1])
}
uReq := &provider.InitiateFileUploadRequest{
Ref: ref,
Opaque: &typespb.Opaque{Map: opaqueMap},
Opaque: opaque,
}
if ifMatch := r.Header.Get(net.HeaderIfMatch); ifMatch != "" {
uReq.Options = &provider.InitiateFileUploadRequest_IfMatch{IfMatch: ifMatch}

View File

@@ -165,7 +165,7 @@ func (fs *cephfs) CreateDir(ctx context.Context, ref *provider.Reference) error
}
// TouchFile as defined in the storage.FS interface
func (fs *cephfs) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool) error {
func (fs *cephfs) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error {
return fmt.Errorf("unimplemented: TouchFile")
}

View File

@@ -266,7 +266,7 @@ func (nc *StorageDriver) CreateDir(ctx context.Context, ref *provider.Reference)
}
// TouchFile as defined in the storage.FS interface
func (nc *StorageDriver) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool) error {
func (nc *StorageDriver) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error {
return fmt.Errorf("unimplemented: TouchFile")
}

View File

@@ -794,7 +794,7 @@ func (fs *owncloudsqlfs) CreateDir(ctx context.Context, ref *provider.Reference)
}
// TouchFile as defined in the storage.FS interface
func (fs *owncloudsqlfs) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool) error {
func (fs *owncloudsqlfs) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error {
ip, err := fs.resolve(ctx, ref)
if err != nil {
return err
@@ -830,15 +830,25 @@ func (fs *owncloudsqlfs) TouchFile(ctx context.Context, ref *provider.Reference,
if err != nil {
return err
}
mtime := time.Now().Unix()
storageMtime := time.Now().Unix()
mt := storageMtime
if mtime != "" {
t, err := strconv.Atoi(mtime)
if err != nil {
log.Info().
Str("owncloudsql", ip).
Msg("error mtime conversion. mtine set to system time")
}
mt = time.Unix(int64(t), 0).Unix()
}
data := map[string]interface{}{
"path": fs.toDatabasePath(ip),
"etag": calcEtag(ctx, fi),
"mimetype": mime.Detect(false, ip),
"permissions": int(conversions.RoleFromResourcePermissions(parentPerms, false).OCSPermissions()), // inherit permissions of parent
"mtime": mtime,
"storage_mtime": mtime,
"mtime": mt,
"storage_mtime": storageMtime,
}
storageID, err := fs.getStorage(ctx, ip)
if err != nil {

View File

@@ -348,7 +348,7 @@ func (fs *s3FS) CreateDir(ctx context.Context, ref *provider.Reference) error {
}
// TouchFile as defined in the storage.FS interface
func (fs *s3FS) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool) error {
func (fs *s3FS) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error {
return fmt.Errorf("unimplemented: TouchFile")
}

View File

@@ -38,7 +38,7 @@ type FS interface {
GetHome(ctx context.Context) (string, error)
CreateHome(ctx context.Context) error
CreateDir(ctx context.Context, ref *provider.Reference) error
TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool) error
TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error
Delete(ctx context.Context, ref *provider.Reference) error
Move(ctx context.Context, oldRef, newRef *provider.Reference) error
GetMD(ctx context.Context, ref *provider.Reference, mdKeys, fieldMask []string) (*provider.ResourceInfo, error)

View File

@@ -76,7 +76,7 @@ type Tree interface {
ListFolder(ctx context.Context, node *node.Node) ([]*node.Node, error)
// CreateHome(owner *userpb.UserId) (n *node.Node, err error)
CreateDir(ctx context.Context, node *node.Node) (err error)
TouchFile(ctx context.Context, node *node.Node, markprocessing bool) error
TouchFile(ctx context.Context, node *node.Node, markprocessing bool, mtime string) error
// CreateReference(ctx context.Context, node *node.Node, targetURI *url.URL) error
Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) (err error)
Delete(ctx context.Context, node *node.Node) (err error)
@@ -616,7 +616,7 @@ func (fs *Decomposedfs) CreateDir(ctx context.Context, ref *provider.Reference)
}
// TouchFile as defined in the storage.FS interface
func (fs *Decomposedfs) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool) error {
func (fs *Decomposedfs) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error {
parentRef := &provider.Reference{
ResourceId: ref.ResourceId,
Path: path.Dir(ref.Path),
@@ -655,7 +655,7 @@ func (fs *Decomposedfs) TouchFile(ctx context.Context, ref *provider.Reference,
if err := n.CheckLock(ctx); err != nil {
return err
}
return fs.tp.TouchFile(ctx, n, markprocessing)
return fs.tp.TouchFile(ctx, n, markprocessing, mtime)
}
// CreateReference creates a reference as a node folder with the target stored in extended attributes

View File

@@ -128,7 +128,7 @@ func (t *Tree) GetMD(ctx context.Context, n *node.Node) (os.FileInfo, error) {
}
// TouchFile creates a new empty file
func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool) error {
func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool, mtime string) error {
if n.Exists {
if markprocessing {
return n.SetXattr(prefixes.StatusPrefix, []byte(node.ProcessingStatus))
@@ -155,6 +155,11 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool)
if markprocessing {
attributes[prefixes.StatusPrefix] = []byte(node.ProcessingStatus)
}
if mtime != "" {
if err := n.SetMtimeString(mtime); err != nil {
return errors.Wrap(err, "Decomposedfs: could not set mtime")
}
}
err = n.SetXattrs(attributes, true)
if err != nil {
return err

View File

@@ -21,6 +21,7 @@ package eosfs
import (
"context"
"database/sql"
b64 "encoding/base64"
"encoding/json"
"fmt"
"io"
@@ -33,8 +34,6 @@ import (
"strings"
"time"
b64 "encoding/base64"
"github.com/bluele/gcache"
grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
@@ -1673,7 +1672,7 @@ func (fs *eosfs) CreateDir(ctx context.Context, ref *provider.Reference) error {
}
// TouchFile as defined in the storage.FS interface
func (fs *eosfs) TouchFile(ctx context.Context, ref *provider.Reference, _ bool) error {
func (fs *eosfs) TouchFile(ctx context.Context, ref *provider.Reference, _ bool, _ string) error {
log := appctx.GetLogger(ctx)
fn, auth, err := fs.resolveRefAndGetAuth(ctx, ref)

View File

@@ -806,7 +806,7 @@ func (fs *localfs) CreateDir(ctx context.Context, ref *provider.Reference) error
}
// TouchFile as defined in the storage.FS interface
func (fs *localfs) TouchFile(ctx context.Context, ref *provider.Reference, _ bool) error {
func (fs *localfs) TouchFile(ctx context.Context, ref *provider.Reference, _ bool, _ string) error {
return fmt.Errorf("unimplemented: TouchFile")
}

2
vendor/modules.txt vendored
View File

@@ -352,7 +352,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1
github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1
github.com/cs3org/go-cs3apis/cs3/tx/v1beta1
github.com/cs3org/go-cs3apis/cs3/types/v1beta1
# github.com/cs3org/reva/v2 v2.14.1-0.20230623085734-919a9585f147 => github.com/micbar/reva/v2 v2.0.0-20230626125956-c381fe19a108
# github.com/cs3org/reva/v2 v2.14.1-0.20230623085734-919a9585f147
## explicit; go 1.20
github.com/cs3org/reva/v2/cmd/revad/internal/grace
github.com/cs3org/reva/v2/cmd/revad/runtime