mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-01-06 04:09:40 -06:00
Code Issues Packages Projects Releases Wiki Activity

update to latest changes in reva PR

Browse Source 
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
Jörn Friedrich Dreyer
2021-11-16 14:20:23 +00:00
parent b63b322b6f
commit c36e99895b
10 changed files with 92 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
storage/pkg/command/frontend.go
View File

@@ -199,6 +199,7 @@ func frontendConfigFromStruct(c *cli.Context, cfg *config.Config, filesCfg map[s
"resource_info_cache_ttl": cfg.Reva.Frontend.OCSResourceInfoCacheTTL,
"prefix": cfg.Reva.Frontend.OCSPrefix,
"additional_info_attribute": cfg.Reva.Frontend.OCSAdditionalInfoAttribute,
"machine_auth_apikey": "change-me-please", // FIXME make configurable
"cache_warmup_driver": cfg.Reva.Frontend.OCSCacheWarmupDriver,
"cache_warmup_drivers": map[string]interface{}{
"cbox": map[string]interface{}{

70
storage/pkg/command/gateway.go
View File

@@ -179,10 +179,6 @@ func gatewayConfigFromStruct(c *cli.Context, cfg *config.Config, logger log.Logg
"storageregistry": map[string]interface{}{
"driver": cfg.Reva.StorageRegistry.Driver,
"drivers": map[string]interface{}{
"static": map[string]interface{}{
"home_provider": cfg.Reva.StorageRegistry.HomeProvider,
"rules": simpleRules(cfg, logger),
},
"spaces": map[string]interface{}{
"home_provider": cfg.Reva.StorageRegistry.HomeProvider,
"rules": spacesRules(cfg, logger),
@@ -195,48 +191,6 @@ func gatewayConfigFromStruct(c *cli.Context, cfg *config.Config, logger log.Logg
return rcfg
}
func simpleRules(cfg *config.Config, logger log.Logger) map[string]map[string]interface{} {
// if a list of rules is given it overrides the generated rules from below
if len(cfg.Reva.StorageRegistry.Rules) > 0 {
rules := map[string]map[string]interface{}{}
for i := range cfg.Reva.StorageRegistry.Rules {
parts := strings.SplitN(cfg.Reva.StorageRegistry.Rules[i], "=", 2)
rules[parts[0]] = map[string]interface{}{"address": parts[1]}
}
return rules
}
// check if the rules have to be read from a json file
if cfg.Reva.StorageRegistry.JSON != "" {
data, err := ioutil.ReadFile(cfg.Reva.StorageRegistry.JSON)
if err != nil {
logger.Error().Err(err).Msg("Failed to read storage registry rules from JSON file: " + cfg.Reva.StorageRegistry.JSON)
return nil
}
var rules map[string]map[string]interface{}
if err = json.Unmarshal(data, &rules); err != nil {
logger.Error().Err(err).Msg("Failed to unmarshal storage registry rules")
return nil
}
return rules
}
// generate rules based on default config
ret := map[string]map[string]interface{}{
cfg.Reva.StorageHome.MountPath: {"address": cfg.Reva.StorageHome.Endpoint},
cfg.Reva.StorageHome.AlternativeID: {"address": cfg.Reva.StorageHome.Endpoint},
cfg.Reva.StorageUsers.MountPath: {"address": cfg.Reva.StorageUsers.Endpoint},
cfg.Reva.StorageUsers.MountID + ".*": {"address": cfg.Reva.StorageUsers.Endpoint},
cfg.Reva.StoragePublicLink.MountPath: {"address": cfg.Reva.StoragePublicLink.Endpoint},
cfg.Reva.StoragePublicLink.MountID: {"address": cfg.Reva.StoragePublicLink.Endpoint},
// public link storage returns the mount id of the actual storage
// medatada storage not part of the global namespace
}
return ret
}
func spacesRules(cfg *config.Config, logger log.Logger) map[string]map[string]interface{} {
// if a list of rules is given it overrides the generated rules from below
@@ -266,11 +220,27 @@ func spacesRules(cfg *config.Config, logger log.Logger) map[string]map[string]in
// generate rules based on default config
return map[string]map[string]interface{}{
"/personal": {"address": cfg.Reva.StorageUsers.Endpoint},
"/home": {
"address": cfg.Reva.StorageUsers.Endpoint,
"space_type": "personal",
"space_owner_self": true,
},
"/home/Shares": {
"address": cfg.Reva.StorageShares.Endpoint,
"space_type": "share",
"path_template": "/home/Shares/{{.Name}}",
},
"/users": {
"address": cfg.Reva.StorageUsers.Endpoint,
"space_type": "personal",
"path_template": "/users/{{.Owner.Id.OpaqueId}}",
},
// public link storage returns the mount id of the actual storage
"/public": {"address": cfg.Reva.StoragePublicLink.Endpoint},
// TODO shares
//"/shares": {"address": cfg.Reva.StoragePublicLink.Endpoint},
"/public": {
"address": cfg.Reva.StoragePublicLink.Endpoint,
"space_type": "public",
"path_template": "/public",
},
// medatada storage not part of the global namespace
}
}

2
storage/pkg/command/root.go
View File

@@ -36,8 +36,8 @@ func Execute(cfg *config.Config) error {
AuthBasic(cfg),
AuthBearer(cfg),
Sharing(cfg),
StorageHome(cfg),
StorageUsers(cfg),
StorageShares(cfg),
StoragePublicLink(cfg),
StorageMetadata(cfg),
Health(cfg),

132
storage/pkg/command/storagedrivers/home.go
View File

@@ -1,132 +0,0 @@
package storagedrivers
import (
"github.com/owncloud/ocis/storage/pkg/config"
)
func HomeDrivers(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{
"eos": map[string]interface{}{
"namespace": cfg.Reva.UserStorage.EOS.Root,
"shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
"uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace,
"share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
"eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.UserStorage.EOS.MasterURL,
"slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
"keytab": cfg.Reva.UserStorage.EOS.Keytab,
"single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
"enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
"gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
},
"eoshome": map[string]interface{}{
"namespace": cfg.Reva.UserStorage.EOS.Root,
"shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
"uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace,
"share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
"eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.UserStorage.EOS.MasterURL,
"slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
"keytab": cfg.Reva.UserStorage.EOS.Keytab,
"single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
"user_layout": cfg.Reva.UserStorage.EOS.UserLayout,
"enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
"gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
},
"eosgrpc": map[string]interface{}{
"namespace": cfg.Reva.UserStorage.EOS.Root,
"shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
"share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
"eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.UserStorage.EOS.MasterURL,
"master_grpc_uri": cfg.Reva.UserStorage.EOS.GrpcURI,
"slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
"keytab": cfg.Reva.UserStorage.EOS.Keytab,
"single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
"user_layout": cfg.Reva.UserStorage.EOS.UserLayout,
"enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
"enable_home": true,
"gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
},
"local": map[string]interface{}{
"root": cfg.Reva.UserStorage.Local.Root,
"share_folder": cfg.Reva.UserStorage.Local.ShareFolder,
},
"localhome": map[string]interface{}{
"root": cfg.Reva.UserStorage.Local.Root,
"share_folder": cfg.Reva.UserStorage.Local.ShareFolder,
"user_layout": cfg.Reva.UserStorage.Local.UserLayout,
},
"owncloud": map[string]interface{}{
"datadirectory": cfg.Reva.UserStorage.OwnCloud.Root,
"upload_info_dir": cfg.Reva.UserStorage.OwnCloud.UploadInfoDir,
"share_folder": cfg.Reva.UserStorage.OwnCloud.ShareFolder,
"user_layout": cfg.Reva.UserStorage.OwnCloud.UserLayout,
"redis": cfg.Reva.UserStorage.OwnCloud.Redis,
"enable_home": true,
"scan": cfg.Reva.UserStorage.OwnCloud.Scan,
"userprovidersvc": cfg.Reva.Users.Endpoint,
},
"owncloudsql": map[string]interface{}{
"datadirectory": cfg.Reva.UserStorage.OwnCloudSQL.Root,
"upload_info_dir": cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir,
"share_folder": cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder,
"user_layout": cfg.Reva.UserStorage.OwnCloudSQL.UserLayout,
"enable_home": true,
"dbusername": cfg.Reva.UserStorage.OwnCloudSQL.DBUsername,
"dbpassword": cfg.Reva.UserStorage.OwnCloudSQL.DBPassword,
"dbhost": cfg.Reva.UserStorage.OwnCloudSQL.DBHost,
"dbport": cfg.Reva.UserStorage.OwnCloudSQL.DBPort,
"dbname": cfg.Reva.UserStorage.OwnCloudSQL.DBName,
"userprovidersvc": cfg.Reva.Users.Endpoint,
},
"ocis": map[string]interface{}{
"root": cfg.Reva.UserStorage.OCIS.Root,
"enable_home": true,
"user_layout": cfg.Reva.UserStorage.OCIS.UserLayout,
"share_folder": cfg.Reva.UserStorage.OCIS.ShareFolder,
"treetime_accounting": true,
"treesize_accounting": true,
"owner": cfg.Reva.UserStorage.OCIS.ServiceUserUUID, // the accounts service system account uuid
},
"s3": map[string]interface{}{
"region": cfg.Reva.UserStorage.S3.Region,
"access_key": cfg.Reva.UserStorage.S3.AccessKey,
"secret_key": cfg.Reva.UserStorage.S3.SecretKey,
"endpoint": cfg.Reva.UserStorage.S3.Endpoint,
"bucket": cfg.Reva.UserStorage.S3.Bucket,
},
"s3ng": map[string]interface{}{
"root": cfg.Reva.UserStorage.S3NG.Root,
"enable_home": true,
"user_layout": cfg.Reva.UserStorage.S3NG.UserLayout,
"treetime_accounting": true,
"treesize_accounting": true,
"owner": cfg.Reva.UserStorage.S3NG.ServiceUserUUID, // the accounts service system account uuid
"share_folder": cfg.Reva.UserStorage.S3NG.ShareFolder,
"s3.region": cfg.Reva.UserStorage.S3NG.Region,
"s3.access_key": cfg.Reva.UserStorage.S3NG.AccessKey,
"s3.secret_key": cfg.Reva.UserStorage.S3NG.SecretKey,
"s3.endpoint": cfg.Reva.UserStorage.S3NG.Endpoint,
"s3.bucket": cfg.Reva.UserStorage.S3NG.Bucket,
},
}
}

1
storage/pkg/command/storagepubliclink.go
View File

@@ -101,6 +101,7 @@ func storagePublicLinkConfigFromStruct(c *cli.Context, cfg *config.Config) map[s
},
"services": map[string]interface{}{
"publicstorageprovider": map[string]interface{}{
"mount_id": cfg.Reva.StoragePublicLink.MountID,
"gateway_addr": cfg.Reva.Gateway.Endpoint,
},
"authprovider": map[string]interface{}{

77
storage/pkg/command/storagehome.go → storage/pkg/command/storageshares.go
View File

@@ -12,7 +12,6 @@ import (
"github.com/gofrs/uuid"
"github.com/oklog/run"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/storage/pkg/command/storagedrivers"
"github.com/owncloud/ocis/storage/pkg/config"
"github.com/owncloud/ocis/storage/pkg/server/debug"
"github.com/owncloud/ocis/storage/pkg/tracing"
@@ -20,13 +19,13 @@ import (
"github.com/urfave/cli/v2"
)
// StorageHome is the entrypoint for the storage-home command.
func StorageHome(cfg *config.Config) *cli.Command {
// StorageShares is the entrypoint for the storage-shares command.
func StorageShares(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "storage-home",
Usage: "Start storage-home service",
Name: "storage-shares",
Usage: "Start storage-shares service",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-home")
return ParseConfig(c, cfg, "storage-shares")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
@@ -40,7 +39,7 @@ func StorageHome(cfg *config.Config) *cli.Command {
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
rcfg := storageHomeConfigFromStruct(c, cfg)
rcfg := storageSharesConfigFromStruct(c, cfg)
gr.Add(func() error {
runtime.RunWithOptions(
@@ -59,7 +58,7 @@ func StorageHome(cfg *config.Config) *cli.Command {
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.StorageHome.DebugAddr),
debug.Addr(cfg.Reva.StorageShares.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
@@ -74,7 +73,7 @@ func StorageHome(cfg *config.Config) *cli.Command {
cancel()
})
if !cfg.Reva.StorageHome.Supervised {
if !cfg.Reva.StorageShares.Supervised {
sync.Trap(&gr, cancel)
}
@@ -83,11 +82,11 @@ func StorageHome(cfg *config.Config) *cli.Command {
}
}
// storageHomeConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func storageHomeConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
// storageSharesConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func storageSharesConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.StorageHome.MaxCPUs,
"max_cpus": cfg.Reva.StorageShares.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
@@ -99,37 +98,17 @@ func storageHomeConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.StorageHome.GRPCNetwork,
"address": cfg.Reva.StorageHome.GRPCAddr,
// TODO build services dynamically
"network": cfg.Reva.StorageShares.GRPCNetwork,
"address": cfg.Reva.StorageShares.GRPCAddr,
"services": map[string]interface{}{
"storageprovider": map[string]interface{}{
"driver": cfg.Reva.StorageHome.Driver,
"drivers": storagedrivers.HomeDrivers(cfg),
"mount_id": cfg.Reva.StorageHome.MountID,
"expose_data_server": cfg.Reva.StorageHome.ExposeDataServer,
"data_server_url": cfg.Reva.StorageHome.DataServerURL,
"tmp_folder": cfg.Reva.StorageHome.TempFolder,
},
},
},
"http": map[string]interface{}{
"network": cfg.Reva.StorageHome.HTTPNetwork,
"address": cfg.Reva.StorageHome.HTTPAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"dataprovider": map[string]interface{}{
"prefix": cfg.Reva.StorageHome.HTTPPrefix,
"driver": cfg.Reva.StorageHome.Driver,
"drivers": storagedrivers.HomeDrivers(cfg),
"timeout": 86400,
"insecure": cfg.Reva.StorageHome.DataProvider.Insecure,
"disable_tus": false,
"sharesstorageprovider": map[string]interface{}{
"usershareprovidersvc": cfg.Reva.Sharing.Endpoint,
"gateway_addr": cfg.Reva.Gateway.Endpoint,
},
},
},
}
if cfg.Reva.StorageHome.ReadOnly {
if cfg.Reva.StorageShares.ReadOnly {
gcfg := rcfg["grpc"].(map[string]interface{})
gcfg["interceptors"] = map[string]interface{}{
"readonly": map[string]interface{}{},
@@ -138,35 +117,35 @@ func storageHomeConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]
return rcfg
}
// StorageHomeSutureService allows for the storage-home command to be embedded and supervised by a suture supervisor tree.
type StorageHomeSutureService struct {
// StorageSharesSutureService allows for the storage-home command to be embedded and supervised by a suture supervisor tree.
type StorageSharesSutureService struct {
cfg *config.Config
}
// NewStorageHomeSutureService creates a new storage.StorageHomeSutureService
func NewStorageHome(cfg *ociscfg.Config) suture.Service {
// NewStorageShares creates a new storage.StorageSharesSutureService
func NewStorageShares(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return StorageHomeSutureService{
return StorageSharesSutureService{
cfg: cfg.Storage,
}
}
func (s StorageHomeSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.StorageHome.Context = ctx
func (s StorageSharesSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.StorageShares.Context = ctx
f := &flag.FlagSet{}
cmdFlags := StorageHome(s.cfg).Flags
cmdFlags := StorageShares(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if StorageHome(s.cfg).Before != nil {
if err := StorageHome(s.cfg).Before(cliCtx); err != nil {
if StorageShares(s.cfg).Before != nil {
if err := StorageShares(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := StorageHome(s.cfg).Action(cliCtx); err != nil {
if err := StorageShares(s.cfg).Action(cliCtx); err != nil {
return err
}