From 6eaa04ee9e97dfd0ca5ba909823043b758f17271 Mon Sep 17 00:00:00 2001 From: Benedikt Kulmann Date: Fri, 23 Oct 2020 16:09:00 +0200 Subject: [PATCH] Rebuild groups index --- accounts/pkg/service/v0/index.go | 21 ++++++--- accounts/pkg/storage/cs3.go | 76 +++++++++++++++++++++++--------- accounts/pkg/storage/disk.go | 19 ++++++-- accounts/pkg/storage/repo.go | 5 +++ 4 files changed, 91 insertions(+), 30 deletions(-) diff --git a/accounts/pkg/service/v0/index.go b/accounts/pkg/service/v0/index.go index 5dab75fac3..dd6184d7b5 100644 --- a/accounts/pkg/service/v0/index.go +++ b/accounts/pkg/service/v0/index.go @@ -21,20 +21,31 @@ func (s Service) RebuildIndex(ctx context.Context, request *proto.RebuildIndexRe return err } - resp := &storage.Accounts{ + accounts := &storage.Accounts{ Accounts: make([]*proto.Account, 0), } - if err := s.repo.LoadAccounts(ctx, resp); err != nil { + if err := s.repo.LoadAccounts(ctx, accounts); err != nil { return err } - for i := range resp.Accounts { - _, err := s.index.Add(resp.Accounts[i]) + for i := range accounts.Accounts { + _, err := s.index.Add(accounts.Accounts[i]) if err != nil { return err } } - // TODO: read all the documents and add them to the indexer. + groups := &storage.Groups{ + Groups: make([]*proto.Group, 0), + } + if err := s.repo.LoadGroups(ctx, groups); err != nil { + return err + } + for i := range groups.Groups { + _, err := s.index.Add(groups.Groups[i]) + if err != nil { + return err + } + } return nil } diff --git a/accounts/pkg/storage/cs3.go b/accounts/pkg/storage/cs3.go index 1417351a47..0fdbcc3033 100644 --- a/accounts/pkg/storage/cs3.go +++ b/accounts/pkg/storage/cs3.go @@ -95,27 +95,7 @@ func (r CS3Repo) LoadAccount(ctx context.Context, id string, a *proto.Account) ( return r.loadAccount(id, t, a) } -func (r CS3Repo) loadAccount(id string, t string, a *proto.Account) error { - resp, err := r.dataProvider.get(r.accountURL(id), t) - if err != nil { - return err - } - - if resp.StatusCode == http.StatusNotFound { - return ¬FoundErr{"account", id} - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - if err = resp.Body.Close(); err != nil { - return err - } - return json.Unmarshal(b, &a) -} - -// LoadAccounts loads all the accounts from the cs3 api. If ids are given, the result set will be filtered. +// LoadAccounts loads all the accounts from the cs3 api func (r CS3Repo) LoadAccounts(ctx context.Context, a *Accounts) (err error) { t, err := r.authenticate(ctx) if err != nil { @@ -145,6 +125,26 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *Accounts) (err error) { return nil } +func (r CS3Repo) loadAccount(id string, t string, a *proto.Account) error { + resp, err := r.dataProvider.get(r.accountURL(id), t) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusNotFound { + return ¬FoundErr{"account", id} + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + if err = resp.Body.Close(); err != nil { + return err + } + return json.Unmarshal(b, &a) +} + // DeleteAccount deletes an account via cs3 by id func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) { t, err := r.authenticate(ctx) @@ -206,6 +206,40 @@ func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err return err } + return r.loadGroup(id, t, g) +} + +// LoadGroups loads all the groups from the cs3 api +func (r CS3Repo) LoadGroups(ctx context.Context, g *Groups) (err error) { + t, err := r.authenticate(ctx) + if err != nil { + return err + } + + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{Path: path.Join("/meta", groupsFolder)}, + }, + }) + if err != nil { + return err + } + + log := olog.NewLogger(olog.Pretty(r.cfg.Log.Pretty), olog.Color(r.cfg.Log.Color), olog.Level(r.cfg.Log.Level)) + for i := range res.Infos { + grp := &proto.Group{} + err := r.loadGroup(filepath.Base(res.Infos[i].Path), t, grp) + if err != nil { + log.Err(err).Msg("could not load account") + continue + } + g.Groups = append(g.Groups, grp) + } + return nil +} + +func (r CS3Repo) loadGroup(id string, t string, g *proto.Group) error { resp, err := r.dataProvider.get(r.groupURL(id), t) if err != nil { return err diff --git a/accounts/pkg/storage/disk.go b/accounts/pkg/storage/disk.go index f7e62cfcfd..4de84d2b93 100644 --- a/accounts/pkg/storage/disk.go +++ b/accounts/pkg/storage/disk.go @@ -70,7 +70,7 @@ func (r DiskRepo) LoadAccount(ctx context.Context, id string, a *proto.Account) return json.Unmarshal(data, a) } -// LoadAccounts loads all the accounts from the local filesystem. If ids are given, the result set will be filtered. +// LoadAccounts loads all the accounts from the local filesystem func (r DiskRepo) LoadAccounts(ctx context.Context, a *Accounts) (err error) { root := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder) return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { @@ -132,6 +132,20 @@ func (r DiskRepo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err return json.Unmarshal(data, g) } +// LoadGroups loads all the groups from the local filesystem +func (r DiskRepo) LoadGroups(ctx context.Context, g *Groups) (err error) { + root := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder) + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + grp := &proto.Group{} + if err = r.LoadGroup(ctx, filepath.Base(path), grp); err != nil { + r.log.Err(err).Msg("could not load group") + return nil + } + g.Groups = append(g.Groups, grp) + return nil + }) +} + // DeleteGroup from the local filesystem func (r DiskRepo) DeleteGroup(ctx context.Context, id string) (err error) { path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, id) @@ -142,9 +156,6 @@ func (r DiskRepo) DeleteGroup(ctx context.Context, id string) (err error) { } return - - //r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove group") - //return merrors.InternalServerError(r.serviceID, "could not remove group: %v", err.Error()) } // deflateMemberOf replaces the groups of A user with an instance that only contains the id diff --git a/accounts/pkg/storage/repo.go b/accounts/pkg/storage/repo.go index 77a9518270..bb3e1a6a7d 100644 --- a/accounts/pkg/storage/repo.go +++ b/accounts/pkg/storage/repo.go @@ -15,6 +15,10 @@ type Accounts struct { Accounts []*proto.Account } +type Groups struct { + Groups []*proto.Group +} + // Repo defines the storage operations type Repo interface { WriteAccount(ctx context.Context, a *proto.Account) (err error) @@ -23,5 +27,6 @@ type Repo interface { DeleteAccount(ctx context.Context, id string) (err error) WriteGroup(ctx context.Context, g *proto.Group) (err error) LoadGroup(ctx context.Context, id string, g *proto.Group) (err error) + LoadGroups(ctx context.Context, g *Groups) (err error) DeleteGroup(ctx context.Context, id string) (err error) }