rework PR

docs/releasing.md

@@ -0,0 +1,47 @@
+---
+title: "Releasing"
+date: 2020-05-22T00:00:00+00:00
+weight: 60
+geekdocRepo: https://github.com/owncloud/ocis-reva
+geekdocEditPath: edit/master/docs
+geekdocFilePath: releasing.md
+---
+
+{{< toc >}}
+
+To release a new version of ocis-reva, you have to follow a few simple steps.
+
+## Preparation
+
+1. Before releasing, make sure that reva has been [updated to the desired version]({{< ref "updating.md" >}})
+2. Create a new branch e.g. `release-x.x.x` where `x.x.x` is the version you want to release.
+3. Checkout the preparation branch.
+4. Create a new changelog folder and move the unreleased snippets there.
+{{< highlight txt >}}
+mkdir changelog/x.x.x_yyyy-MM-dd/ # yyyy-MM-dd is the current date
+mv changelog/unreleased/* changelog/x.x.x_yyyy-MM-dd/
+{{< / highlight >}}
+5. Commit and push the changes
+{{< highlight txt >}}
+git add --all
+git commit -m "prepare release x.x.x"
+git push origin release-x.x.x
+{{< / highlight >}}
+6. Create a pull request to the master branch.
+
+## Release
+1. After the preparation branch has been merged update your local master.
+2. [Wait for CI](https://cloud.drone.io/owncloud/ocis-reva) to generate a commit for the changelog update
+3. Check out master (or make sure to check out the generated changelog commit in case of subsequent merges)
+{{< highlight txt >}}
+git checkout master
+git pull origin master
+{{< / highlight >}}
+4. Create a new tag (preferably signed) and replace the version number accordingly.
+{{< highlight txt >}}
+git tag -s vx.x.x -m "release vx.x.x"
+git push origin vx.x.x
+{{< / highlight >}}
+5. Wait for CI and check that the GitHub release was published.
+
+Congratulations, you just released ocis-reva!

docs/updating.md

@@ -0,0 +1,19 @@
+---
+title: "Updating reva"
+date: 2020-05-22T00:00:00+00:00
+weight: 50
+geekdocRepo: https://github.com/owncloud/ocis-reva
+geekdocEditPath: edit/master/docs
+geekdocFilePath: updating.md
+---
+
+{{< toc >}}
+
+## Updating reva
+
+1. Run `go get github.com/cs3org/reva@master`
+2. Create a changelog entry containing changes that were done in [reva](https://github.com/cs3org/reva/commits/master)
+3. Create a Pull Request to ocis-reva master with those changes
+4. If test issues appear, you might need to [adjust the tests]({{< ref "testing.md" >}})
+5. After the PR is merged, consider doing a [release of ocis-reva]({{< ref "releasing.md" >}})
+

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/micro/cli/v2 v2.1.1
 	github.com/micro/go-micro v1.18.0
 	github.com/micro/go-micro/v2 v2.0.0
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/oklog/run v1.0.0
 	github.com/owncloud/flaex v0.0.0-20200411150708-dce59891a203
 	github.com/owncloud/ocis-pkg/v2 v2.2.1
@@ -18,11 +19,9 @@ require (
 	github.com/prometheus/client_model v0.0.0-20191202183732-d1d2010b5bee // indirect
 	github.com/prometheus/procfs v0.0.8 // indirect
 	github.com/restic/calens v0.2.0
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.6.1
 	github.com/uber/jaeger-client-go v2.20.1+incompatible // indirect
-	golang.org/x/sys v0.0.0-20200103143344-a1369afcdac7 // indirect
 	gopkg.in/ini.v1 v1.51.1 // indirect
 )
+
+replace github.com/cs3org/reva => ../../refs/reva

pkg/command/frontend.go

@@ -186,7 +186,7 @@ func Frontend(cfg *config.Config) *cli.Command {
 											},
 										},
 										"notifications": map[string]interface{}{
-											"endpoints": []string{"list", "get", "delete"},
+											"endpoints": []string{"disable"},
 										},
 									},
 									"version": map[string]interface{}{

pkg/command/gateway.go

@@ -104,6 +104,8 @@ func Gateway(cfg *config.Config) *cli.Command {
 								"commit_share_to_storage_grant": cfg.Reva.Gateway.CommitShareToStorageGrant,
 								"commit_share_to_storage_ref":   cfg.Reva.Gateway.CommitShareToStorageRef,
 								"share_folder":                  cfg.Reva.Gateway.ShareFolder, // ShareFolder is the location where to create shares in the recipient's storage provider.
+								// public links
+								"link_grants_file": cfg.Reva.Gateway.LinkGrants,
 								// other
 								"disable_home_creation_on_login": cfg.Reva.Gateway.DisableHomeCreationOnLogin,
 								"datagateway":                    cfg.Reva.Frontend.URL,
@@ -115,8 +117,9 @@ func Gateway(cfg *config.Config) *cli.Command {
 								"drivers": map[string]interface{}{
 									"static": map[string]interface{}{
 										"rules": map[string]interface{}{
-											"basic":  cfg.Reva.AuthBasic.URL,
-											"bearer": cfg.Reva.AuthBearer.URL,
+											"basic":        cfg.Reva.AuthBasic.URL,
+											"bearer":       cfg.Reva.AuthBearer.URL,
+											"publicshares": cfg.Reva.StoragePublicLink.URL,
 										},
 									},
 								},
@@ -131,16 +134,18 @@ func Gateway(cfg *config.Config) *cli.Command {
 											cfg.Reva.StorageRoot.MountID:   cfg.Reva.StorageRoot.URL,
 											cfg.Reva.StorageHome.MountPath: cfg.Reva.StorageHome.URL,
 											// the home storage has no mount id. In responses it returns the mount id of the actual storage
-											cfg.Reva.StorageEOS.MountPath:    cfg.Reva.StorageEOS.URL,
-											cfg.Reva.StorageEOS.MountID:      cfg.Reva.StorageEOS.URL,
-											cfg.Reva.StorageOC.MountPath:     cfg.Reva.StorageOC.URL,
-											cfg.Reva.StorageOC.MountID:       cfg.Reva.StorageOC.URL,
-											cfg.Reva.StorageS3.MountPath:     cfg.Reva.StorageS3.URL,
-											cfg.Reva.StorageS3.MountID:       cfg.Reva.StorageS3.URL,
-											cfg.Reva.StorageWND.MountPath:    cfg.Reva.StorageWND.URL,
-											cfg.Reva.StorageWND.MountID:      cfg.Reva.StorageWND.URL,
-											cfg.Reva.StorageCustom.MountPath: cfg.Reva.StorageCustom.URL,
-											cfg.Reva.StorageCustom.MountID:   cfg.Reva.StorageCustom.URL,
+											cfg.Reva.StorageEOS.MountPath:          cfg.Reva.StorageEOS.URL,
+											cfg.Reva.StorageEOS.MountID:            cfg.Reva.StorageEOS.URL,
+											cfg.Reva.StorageOC.MountPath:           cfg.Reva.StorageOC.URL,
+											cfg.Reva.StorageOC.MountID:             cfg.Reva.StorageOC.URL,
+											cfg.Reva.StorageS3.MountPath:           cfg.Reva.StorageS3.URL,
+											cfg.Reva.StorageS3.MountID:             cfg.Reva.StorageS3.URL,
+											cfg.Reva.StorageWND.MountPath:          cfg.Reva.StorageWND.URL,
+											cfg.Reva.StorageWND.MountID:            cfg.Reva.StorageWND.URL,
+											cfg.Reva.StorageCustom.MountPath:       cfg.Reva.StorageCustom.URL,
+											cfg.Reva.StorageCustom.MountID:         cfg.Reva.StorageCustom.URL,
+											"/public/":                             "localhost:10054",
+											"e1a73ede-549b-4226-abdf-40e69ca8230d": "localhost:10054",
 										},
 									},
 								},

pkg/command/root.go

@@ -83,6 +83,7 @@ func Execute() error {
 			StorageRoot(cfg),
 			StorageHome(cfg),
 			StorageHomeData(cfg),
+			StoragePublicLink(cfg),
 			StorageOC(cfg),
 			StorageOCData(cfg),
 			StorageEOS(cfg),

pkg/command/storagepubliclink.go

@@ -0,0 +1,175 @@
+package command
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"path"
+	"time"
+
+	"github.com/cs3org/reva/cmd/revad/runtime"
+	"github.com/gofrs/uuid"
+	"github.com/micro/cli/v2"
+	"github.com/oklog/run"
+	"github.com/owncloud/ocis-reva/pkg/config"
+	"github.com/owncloud/ocis-reva/pkg/flagset"
+	"github.com/owncloud/ocis-reva/pkg/server/debug"
+)
+
+// StoragePublicLink is the entrypoint for the reva-storage-public-link command.
+func StoragePublicLink(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:     "reva-storage-public-link",
+		Usage:    "Start reva storage-public-link service",
+		Flags:    flagset.StoragePublicLink(cfg),
+		Category: "Extensions",
+		Action: func(c *cli.Context) error {
+			logger := NewLogger(cfg)
+
+			if cfg.Tracing.Enabled {
+				switch t := cfg.Tracing.Type; t {
+				case "agent":
+					logger.Error().
+						Str("type", t).
+						Msg("Reva only supports the jaeger tracing backend")
+
+				case "jaeger":
+					logger.Info().
+						Str("type", t).
+						Msg("configuring reva to use the jaeger tracing backend")
+
+				case "zipkin":
+					logger.Error().
+						Str("type", t).
+						Msg("Reva only supports the jaeger tracing backend")
+
+				default:
+					logger.Warn().
+						Str("type", t).
+						Msg("Unknown tracing backend")
+				}
+
+			} else {
+				logger.Debug().
+					Msg("Tracing is not enabled")
+			}
+
+			var (
+				gr          = run.Group{}
+				ctx, cancel = context.WithCancel(context.Background())
+				//metrics     = metrics.New()
+			)
+
+			defer cancel()
+
+			{
+				uuid := uuid.Must(uuid.NewV4())
+				pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
+
+				rcfg := map[string]interface{}{
+					"core": map[string]interface{}{
+						"max_cpus":        cfg.Reva.StoragePublicLink.MaxCPUs,
+						"tracing_enabled": true,
+					},
+					"shared": map[string]interface{}{
+						"jwt_secret": cfg.Reva.JWTSecret,
+					},
+					"grpc": map[string]interface{}{
+						"network": cfg.Reva.StoragePublicLink.Network,
+						"address": cfg.Reva.StoragePublicLink.Addr,
+						"interceptors": map[string]interface{}{
+							"log": map[string]interface{}{},
+						},
+						"services": map[string]interface{}{
+							"publicstorageprovider": map[string]interface{}{
+								"mount_path":   cfg.Reva.StoragePublicLink.MountPath,
+								"mount_id":     cfg.Reva.StoragePublicLink.MountID,
+								"driver_addr":  cfg.Reva.StoragePublicLink.PublicShareProviderAddr,
+								"gateway_addr": cfg.Reva.Gateway.URL,
+							},
+							"authprovider": map[string]interface{}{
+								"auth_manager": "publicshares",
+								"auth_managers": map[string]interface{}{
+									"publicshares": map[string]interface{}{
+										"gateway_addr": cfg.Reva.Gateway.URL,
+									},
+								},
+							},
+						},
+					},
+				}
+
+				gr.Add(func() error {
+					runtime.RunWithOptions(
+						rcfg,
+						pidFile,
+						runtime.WithLogger(&logger.Logger),
+					)
+					return nil
+				}, func(_ error) {
+					logger.Info().
+						Str("server", c.Command.Name).
+						Msg("Shutting down server")
+
+					cancel()
+				})
+			}
+
+			{
+				server, err := debug.Server(
+					debug.Name(c.Command.Name+"-debug"),
+					debug.Addr(cfg.Reva.StoragePublicLink.DebugAddr),
+					debug.Logger(logger),
+					debug.Context(ctx),
+					debug.Config(cfg),
+				)
+
+				if err != nil {
+					logger.Info().
+						Err(err).
+						Str("server", c.Command.Name+"-debug").
+						Msg("Failed to initialize server")
+
+					return err
+				}
+
+				gr.Add(func() error {
+					return server.ListenAndServe()
+				}, func(_ error) {
+					ctx, timeout := context.WithTimeout(ctx, 5*time.Second)
+
+					defer timeout()
+					defer cancel()
+
+					if err := server.Shutdown(ctx); err != nil {
+						logger.Info().
+							Err(err).
+							Str("server", c.Command.Name+"-debug").
+							Msg("Failed to shutdown server")
+					} else {
+						logger.Info().
+							Str("server", c.Command.Name+"-debug").
+							Msg("Shutting down server")
+					}
+				})
+			}
+
+			{
+				stop := make(chan os.Signal, 1)
+
+				gr.Add(func() error {
+					signal.Notify(stop, os.Interrupt)
+
+					<-stop
+
+					return nil
+				}, func(err error) {
+					close(stop)
+					cancel()
+				})
+			}
+
+			return gr.Run()
+		},
+	}
+}

pkg/config/config.go

@@ -21,6 +21,7 @@ type Gateway struct {
 	CommitShareToStorageGrant  bool
 	CommitShareToStorageRef    bool
 	ShareFolder                string
+	LinkGrants                 string
 	DisableHomeCreationOnLogin bool
 	// include the home folder config for the storage registry
 	// HomeProvider is the path in the global namespace that the static storage registry uses to determine the home storage
@@ -79,6 +80,16 @@ type StoragePort struct {
 	TempFolder string
 }
 
+// PublicStorage configures a public storage provider
+type PublicStorage struct {
+	StoragePort
+
+	PublicShareProviderAddr string
+	StorageProviderAddr     string
+	UserProviderAddr        string
+	MountID                 string
+}
+
 // StorageConfig combines all available storage driver configuration parts.
 type StorageConfig struct {
 	EOS      DriverEOS
@@ -238,6 +249,7 @@ type Reva struct {
 	StorageWNDData    StoragePort
 	StorageCustom     StoragePort
 	StorageCustomData StoragePort
+	StoragePublicLink PublicStorage
 	// Configs can be used to configure the reva instance.
 	// Services and Ports will be ignored if this is used
 	Configs map[string]interface{}

pkg/flagset/gateway.go

@@ -159,6 +159,14 @@ func GatewayWithConfig(cfg *config.Config) []cli.Flag {
 			EnvVars:     []string{"REVA_GATEWAY_SHARE_FOLDER"},
 			Destination: &cfg.Reva.Gateway.ShareFolder,
 		},
+		// TODO(refs) temporary workaround needed for storing link grants.
+		&cli.StringFlag{
+			Name:        "link_grants_file",
+			Value:       "/var/tmp/reva/link_grants.json",
+			Usage:       "when using a json manager, file to use as json serialized database",
+			EnvVars:     []string{"REVA_GATEWAY_LINK_GRANTS_FILE"},
+			Destination: &cfg.Reva.Gateway.LinkGrants,
+		},
 		&cli.BoolFlag{
 			Name:        "disable-home-creation-on-login",
 			Usage:       "Disable creation of home folder on login",
@@ -295,5 +303,12 @@ func GatewayWithConfig(cfg *config.Config) []cli.Flag {
 			EnvVars:     []string{"REVA_STORAGE_OC_MOUNT_ID"},
 			Destination: &cfg.Reva.StorageOC.MountID,
 		},
+		&cli.StringFlag{
+			Name:        "public-links-url",
+			Value:       "localhost:10054",
+			Usage:       "URL to use for the public links service",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_URL"},
+			Destination: &cfg.Reva.StoragePublicLink.URL,
+		},
 	}
 }

pkg/flagset/sharing.go

@@ -136,7 +136,7 @@ func SharingWithConfig(cfg *config.Config) []cli.Flag {
 		},
 		&cli.StringFlag{
 			Name:        "public-driver",
-			Value:       "memory",
+			Value:       "json",
 			Usage:       "driver to use for the PublicShareProvider",
 			EnvVars:     []string{"REVA_SHARING_PUBLIC_DRIVER"},
 			Destination: &cfg.Reva.Sharing.PublicDriver,

pkg/flagset/storagepubliclink.go

@@ -0,0 +1,177 @@
+package flagset
+
+import (
+	"github.com/micro/cli/v2"
+	"github.com/owncloud/ocis-reva/pkg/config"
+)
+
+// StoragePublicLink applies cfg to the root flagset
+func StoragePublicLink(cfg *config.Config) []cli.Flag {
+	return []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "tracing-enabled",
+			Usage:       "Enable sending traces",
+			EnvVars:     []string{"REVA_TRACING_ENABLED"},
+			Destination: &cfg.Tracing.Enabled,
+		},
+		&cli.StringFlag{
+			Name:        "tracing-type",
+			Value:       "jaeger",
+			Usage:       "Tracing backend type",
+			EnvVars:     []string{"REVA_TRACING_TYPE"},
+			Destination: &cfg.Tracing.Type,
+		},
+		&cli.StringFlag{
+			Name:        "tracing-endpoint",
+			Value:       "",
+			Usage:       "Endpoint for the agent",
+			EnvVars:     []string{"REVA_TRACING_ENDPOINT"},
+			Destination: &cfg.Tracing.Endpoint,
+		},
+		&cli.StringFlag{
+			Name:        "tracing-collector",
+			Value:       "",
+			Usage:       "Endpoint for the collector",
+			EnvVars:     []string{"REVA_TRACING_COLLECTOR"},
+			Destination: &cfg.Tracing.Collector,
+		},
+		&cli.StringFlag{
+			Name:        "tracing-service",
+			Value:       "reva",
+			Usage:       "Service name for tracing",
+			EnvVars:     []string{"REVA_TRACING_SERVICE"},
+			Destination: &cfg.Tracing.Service,
+		},
+		&cli.StringFlag{
+			Name:        "debug-addr",
+			Value:       "0.0.0.0:10053",
+			Usage:       "Address to bind debug server",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_DEBUG_ADDR"},
+			Destination: &cfg.Reva.StoragePublicLink.DebugAddr,
+		},
+		&cli.StringFlag{
+			Name:        "debug-token",
+			Value:       "",
+			Usage:       "Token to grant metrics access",
+			EnvVars:     []string{"REVA_DEBUG_TOKEN"},
+			Destination: &cfg.Debug.Token,
+		},
+		&cli.BoolFlag{
+			Name:        "debug-pprof",
+			Usage:       "Enable pprof debugging",
+			EnvVars:     []string{"REVA_DEBUG_PPROF"},
+			Destination: &cfg.Debug.Pprof,
+		},
+		&cli.BoolFlag{
+			Name:        "debug-zpages",
+			Usage:       "Enable zpages debugging",
+			EnvVars:     []string{"REVA_DEBUG_ZPAGES"},
+			Destination: &cfg.Debug.Zpages,
+		},
+		&cli.StringFlag{
+			Name:        "jwt-secret",
+			Value:       "Pive-Fumkiu4",
+			Usage:       "Shared jwt secret for reva service communication",
+			EnvVars:     []string{"REVA_JWT_SECRET"},
+			Destination: &cfg.Reva.JWTSecret,
+		},
+		&cli.StringFlag{
+			Name:        "network",
+			Value:       "tcp",
+			Usage:       "Network to use for the reva service, can be 'tcp', 'udp' or 'unix'",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_NETWORK"},
+			Destination: &cfg.Reva.StoragePublicLink.Network,
+		},
+		&cli.StringFlag{
+			Name:        "protocol",
+			Value:       "grpc",
+			Usage:       "protocol for reva service, can be 'http' or 'grpc'",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_PROTOCOL"},
+			Destination: &cfg.Reva.StoragePublicLink.Protocol,
+		},
+		&cli.StringFlag{
+			Name:        "addr",
+			Value:       "localhost:10054",
+			Usage:       "Address to bind reva service",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_ADDR"},
+			Destination: &cfg.Reva.StoragePublicLink.Addr,
+		},
+		&cli.StringSliceFlag{
+			Name:    "service",
+			Value:   cli.NewStringSlice("storageprovider"),
+			Usage:   "--service storageprovider [--service otherservice]",
+			EnvVars: []string{"REVA_STORAGE_PUBLIC_LINK_SERVICES"},
+		},
+
+		&cli.StringFlag{
+			Name:        "public_share_provider_addr",
+			Value:       "localhost:9150",
+			Usage:       "public share provider service address",
+			EnvVars:     []string{"REVA_STORAGE_PUBLICLINK_PUBLIC_SHARE_PROVIDER_ADDR"},
+			Destination: &cfg.Reva.StoragePublicLink.PublicShareProviderAddr,
+		},
+		&cli.StringFlag{
+			Name:        "user_provider_addr",
+			Value:       "localhost:9144",
+			Usage:       "user provider service address",
+			EnvVars:     []string{"REVA_STORAGE_PUBLICLINK_USER_PROVIDER_ADDR"},
+			Destination: &cfg.Reva.StoragePublicLink.UserProviderAddr,
+		},
+		&cli.StringFlag{
+			Name:        "storage_provider_addr",
+			Value:       "localhost:9154",
+			Usage:       "storage provider service address",
+			EnvVars:     []string{"REVA_STORAGE_PUBLICLINK_STORAGE_PROVIDER_ADDR"},
+			Destination: &cfg.Reva.StoragePublicLink.StorageProviderAddr,
+		},
+		&cli.StringFlag{
+			Name:        "driver",
+			Value:       "owncloud",
+			Usage:       "storage driver, eg. local, eos, owncloud or s3",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_DRIVER"},
+			Destination: &cfg.Reva.StoragePublicLink.Driver,
+		},
+		&cli.StringFlag{
+			Name:        "mount-id",
+			Value:       "e1a73ede-549b-4226-abdf-40e69ca8230d",
+			Usage:       "mount id",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_MOUNT_ID"},
+			Destination: &cfg.Reva.StoragePublicLink.MountID,
+		},
+		&cli.BoolFlag{
+			Name:        "expose-data-server",
+			Value:       true,
+			Usage:       "exposes a dedicated data server",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_EXPOSE_DATA_SERVER"},
+			Destination: &cfg.Reva.StoragePublicLink.ExposeDataServer,
+		},
+		&cli.StringFlag{
+			Name:        "data-server-url",
+			Value:       "http://localhost:9156/data",
+			Usage:       "data server url",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_DATA_SERVER_URL"},
+			Destination: &cfg.Reva.StoragePublicLink.DataServerURL,
+		},
+		&cli.BoolFlag{
+			Name:        "enable-home-creation",
+			Value:       true,
+			Usage:       "if enabled home dirs will be automatically created",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_ENABLE_HOME_CREATION"},
+			Destination: &cfg.Reva.StoragePublicLink.EnableHomeCreation,
+		},
+		&cli.StringFlag{
+			Name:        "mount-path",
+			Value:       "/public/",
+			Usage:       "mount path",
+			EnvVars:     []string{"REVA_STORAGE_PUBLIC_LINK_MOUNT_PATH"},
+			Destination: &cfg.Reva.StoragePublicLink.MountPath,
+		},
+		&cli.StringFlag{
+			Name:        "gateway-url",
+			Value:       "localhost:9142",
+			Usage:       "URL to use for the reva gateway service",
+			EnvVars:     []string{"REVA_GATEWAY_URL"},
+			Destination: &cfg.Reva.Gateway.URL,
+		},
+	}
+}