mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-05-24 22:19:09 -05:00
[full-ci] chore: bump reva to v2.45.0 (#2787)
This commit is contained in:
@@ -64,7 +64,7 @@ require (
|
||||
github.com/open-policy-agent/opa v1.15.2
|
||||
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d
|
||||
github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721
|
||||
github.com/opencloud-eu/reva/v2 v2.45.0
|
||||
github.com/opensearch-project/opensearch-go/v4 v4.6.0
|
||||
github.com/orcaman/concurrent-map v1.0.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
|
||||
@@ -952,8 +952,8 @@ github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9 h1:dIft
|
||||
github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9/go.mod h1:JWyDC6H+5oZRdUJUgKuaye+8Ph5hEs6HVzVoPKzWSGI=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d h1:JcqGDiyrcaQwVyV861TUyQgO7uEmsjkhfm7aQd84dOw=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q=
|
||||
github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721 h1:PH9Ia0HwdvpfaThYCid2atc5y+uwn4EJxvJU6L9wU6M=
|
||||
github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721/go.mod h1:tUL2X47YxLHrnBDArHrIP73UJliMI0PaY/3tPs31dTM=
|
||||
github.com/opencloud-eu/reva/v2 v2.45.0 h1:62XctwzrGKjmWng9QI+tACEDQb6R9fG7oHlMPfQkjNs=
|
||||
github.com/opencloud-eu/reva/v2 v2.45.0/go.mod h1:tUL2X47YxLHrnBDArHrIP73UJliMI0PaY/3tPs31dTM=
|
||||
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
|
||||
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
|
||||
+30
-21
@@ -24,9 +24,9 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
)
|
||||
|
||||
type IDCache struct {
|
||||
@@ -59,17 +59,20 @@ func (c *IDCache) Delete(ctx context.Context, spaceID, nodeID string) error {
|
||||
func (c *IDCache) DeleteByPath(ctx context.Context, path string) error {
|
||||
baseKey := reverseCacheKey(path)
|
||||
|
||||
spaceID, nodeID, ok := c.GetByPath(ctx, path)
|
||||
if !ok {
|
||||
appctx.GetLogger(ctx).Error().Str("record", path).Msg("could not get spaceID and nodeID from cache")
|
||||
spaceID, nodeID, err := c.GetByPath(ctx, path)
|
||||
if err != nil {
|
||||
if _, ok := err.(errtypes.NotFound); !ok {
|
||||
return err
|
||||
}
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("record", path).Msg("could not get spaceID and nodeID from cache")
|
||||
} else {
|
||||
err := c.kv.Purge(ctx, baseKey)
|
||||
if err != nil && err != nats.ErrKeyNotFound {
|
||||
if err != nil && err != jetstream.ErrKeyNotFound {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("record", baseKey).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache")
|
||||
}
|
||||
|
||||
err = c.kv.Purge(ctx, cacheKey(spaceID, nodeID))
|
||||
if err != nil && err != nats.ErrKeyNotFound {
|
||||
if err != nil && err != jetstream.ErrKeyNotFound {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("record", cacheKey(spaceID, nodeID)).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache")
|
||||
}
|
||||
}
|
||||
@@ -85,19 +88,19 @@ func (c *IDCache) DeleteByPath(ctx context.Context, path string) error {
|
||||
break
|
||||
}
|
||||
key := update.Key()
|
||||
spaceID, nodeID, ok := c.getByReverseCacheKey(ctx, key)
|
||||
if !ok {
|
||||
appctx.GetLogger(ctx).Error().Str("record", key).Msg("could not get spaceID and nodeID from cache")
|
||||
spaceID, nodeID, err := c.getByReverseCacheKey(ctx, key)
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("record", key).Msg("could not get spaceID and nodeID from cache")
|
||||
continue
|
||||
}
|
||||
|
||||
err := c.kv.Purge(ctx, key)
|
||||
if err != nil && err != nats.ErrKeyNotFound {
|
||||
err = c.kv.Purge(ctx, key)
|
||||
if err != nil && err != jetstream.ErrKeyNotFound {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("record", key).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache")
|
||||
}
|
||||
|
||||
err = c.kv.Purge(ctx, cacheKey(spaceID, nodeID))
|
||||
if err != nil && err != nats.ErrKeyNotFound {
|
||||
if err != nil && err != jetstream.ErrKeyNotFound {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("record", cacheKey(spaceID, nodeID)).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache")
|
||||
}
|
||||
}
|
||||
@@ -121,32 +124,38 @@ func (c *IDCache) Set(ctx context.Context, spaceID, nodeID, val string) error {
|
||||
}
|
||||
|
||||
// Get returns the value for a given key
|
||||
func (c *IDCache) Get(ctx context.Context, spaceID, nodeID string) (string, bool) {
|
||||
func (c *IDCache) Get(ctx context.Context, spaceID, nodeID string) (string, error) {
|
||||
record, err := c.kv.Get(ctx, cacheKey(spaceID, nodeID))
|
||||
if err != nil {
|
||||
return "", false
|
||||
if err == jetstream.ErrKeyNotFound {
|
||||
return "", errtypes.NotFound("record not found in cache")
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
return string(record.Value()), true
|
||||
return string(record.Value()), nil
|
||||
}
|
||||
|
||||
func (c *IDCache) getByReverseCacheKey(ctx context.Context, reverseKey string) (string, string, bool) {
|
||||
func (c *IDCache) getByReverseCacheKey(ctx context.Context, reverseKey string) (string, string, error) {
|
||||
record, err := c.kv.Get(ctx, reverseKey)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
if err == jetstream.ErrKeyNotFound {
|
||||
return "", "", errtypes.NotFound("record not found in cache")
|
||||
}
|
||||
return "", "", err
|
||||
}
|
||||
decoded, err := base32.StdEncoding.DecodeString(string(record.Value()))
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
return "", "", err
|
||||
}
|
||||
parts := strings.SplitN(string(decoded), "!", 2)
|
||||
if len(parts) != 2 {
|
||||
return "", "", false
|
||||
return "", "", errtypes.InternalError("invalid cache record")
|
||||
}
|
||||
return parts[0], parts[1], true
|
||||
return parts[0], parts[1], nil
|
||||
}
|
||||
|
||||
// GetByPath returns the key for a given value
|
||||
func (c *IDCache) GetByPath(ctx context.Context, path string) (string, string, bool) {
|
||||
func (c *IDCache) GetByPath(ctx context.Context, path string) (string, string, error) {
|
||||
return c.getByReverseCacheKey(ctx, reverseCacheKey(path))
|
||||
}
|
||||
|
||||
|
||||
+32
-18
@@ -41,6 +41,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"github.com/rs/zerolog"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
@@ -58,8 +59,8 @@ func init() {
|
||||
|
||||
// IDCache is a cache for node ids
|
||||
type IDCache interface {
|
||||
Get(ctx context.Context, spaceID, nodeID string) (string, bool)
|
||||
GetByPath(ctx context.Context, path string) (string, string, bool)
|
||||
Get(ctx context.Context, spaceID, nodeID string) (string, error)
|
||||
GetByPath(ctx context.Context, path string) (string, string, error)
|
||||
|
||||
Set(ctx context.Context, spaceID, nodeID, val string) error
|
||||
|
||||
@@ -79,10 +80,12 @@ type Lookup struct {
|
||||
metadataBackend metadata.Backend
|
||||
userMapper usermapper.Mapper
|
||||
tm node.TimeManager
|
||||
log *zerolog.Logger
|
||||
}
|
||||
|
||||
// New returns a new Lookup instance
|
||||
func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.TimeManager, cache, historyCache *idcache.IDCache) (*Lookup, error) {
|
||||
func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.TimeManager, cache, historyCache *idcache.IDCache, log *zerolog.Logger) (*Lookup, error) {
|
||||
|
||||
spaceRootCache, _ := lru.New[string, string](1000)
|
||||
|
||||
lu := &Lookup{
|
||||
@@ -93,6 +96,7 @@ func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.T
|
||||
spaceRootCache: spaceRootCache,
|
||||
userMapper: um,
|
||||
tm: tm,
|
||||
log: log,
|
||||
}
|
||||
|
||||
return lu, nil
|
||||
@@ -107,7 +111,7 @@ func (lu *Lookup) CacheID(ctx context.Context, spaceID, nodeID, val string) erro
|
||||
}
|
||||
|
||||
// GetCachedID returns the cached path for the given space and node id
|
||||
func (lu *Lookup) GetCachedID(ctx context.Context, spaceID, nodeID string) (string, bool) {
|
||||
func (lu *Lookup) GetCachedID(ctx context.Context, spaceID, nodeID string) (string, error) {
|
||||
if spaceID == nodeID {
|
||||
return lu.getSpaceRootPathWithStatus(ctx, spaceID)
|
||||
}
|
||||
@@ -116,18 +120,26 @@ func (lu *Lookup) GetCachedID(ctx context.Context, spaceID, nodeID string) (stri
|
||||
|
||||
func (lu *Lookup) IDsForPath(ctx context.Context, path string) (string, string, error) {
|
||||
// IDsForPath returns the space and opaque id for the given path
|
||||
spaceID, nodeID, ok := lu.IDCache.GetByPath(ctx, path)
|
||||
if !ok {
|
||||
return "", "", errtypes.NotFound("path not found in cache:" + path)
|
||||
spaceID, nodeID, err := lu.IDCache.GetByPath(ctx, path)
|
||||
if err != nil {
|
||||
if _, ok := err.(errtypes.NotFound); !ok {
|
||||
lu.log.Error().Err(err).Str("path", path).Msg("error looking up path in cache")
|
||||
}
|
||||
// fallback to disk
|
||||
sID, nID, _, _, mErr := lu.metadataBackend.IdentifyPath(ctx, path)
|
||||
if mErr == nil && nID != "" {
|
||||
return sID, nID, nil
|
||||
}
|
||||
return "", "", err
|
||||
}
|
||||
return spaceID, nodeID, nil
|
||||
}
|
||||
|
||||
// NodeFromPath returns the node for the given path
|
||||
func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) {
|
||||
parentPath, ok := lu.GetCachedID(ctx, parent.SpaceID, parent.ID)
|
||||
if !ok {
|
||||
return "", errtypes.NotFound(parent.ID)
|
||||
parentPath, err := lu.GetCachedID(ctx, parent.SpaceID, parent.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
childPath := filepath.Join(parentPath, name)
|
||||
@@ -290,15 +302,15 @@ func (lu *Lookup) InternalRoot() string {
|
||||
return lu.Options.Root
|
||||
}
|
||||
|
||||
func (lu *Lookup) getSpaceRootPathWithStatus(ctx context.Context, spaceID string) (string, bool) {
|
||||
func (lu *Lookup) getSpaceRootPathWithStatus(ctx context.Context, spaceID string) (string, error) {
|
||||
if val, ok := lu.spaceRootCache.Get(spaceID); ok {
|
||||
return val, true
|
||||
return val, nil
|
||||
}
|
||||
val, ok := lu.IDCache.Get(ctx, spaceID, spaceID)
|
||||
if ok {
|
||||
val, err := lu.IDCache.Get(ctx, spaceID, spaceID)
|
||||
if err == nil {
|
||||
lu.spaceRootCache.Add(spaceID, val)
|
||||
}
|
||||
return val, ok
|
||||
return val, err
|
||||
}
|
||||
|
||||
func (lu *Lookup) getSpaceRootPath(ctx context.Context, spaceID string) string {
|
||||
@@ -338,9 +350,11 @@ func (lu *Lookup) LockfilePaths(n *node.Node) []string {
|
||||
}
|
||||
paths := []string{filepath.Join(spaceRoot, MetadataDir, Pathify(n.ID, 4, 2)+".lock")}
|
||||
|
||||
nodepath := n.InternalPath()
|
||||
if len(nodepath) > 0 {
|
||||
paths = append(paths, nodepath+".lock")
|
||||
if lu.Options.WatchFS {
|
||||
nodepath := n.InternalPath()
|
||||
if len(nodepath) > 0 {
|
||||
paths = append(paths, nodepath+".lock")
|
||||
}
|
||||
}
|
||||
|
||||
return paths
|
||||
|
||||
+2
-2
@@ -122,7 +122,7 @@ func New(o *options.Options, stream events.Stream, cache, historyCache *idcache.
|
||||
var lu *lookup.Lookup
|
||||
switch o.MetadataBackend {
|
||||
case "xattrs":
|
||||
lu, err = lookup.New(metadata.NewXattrsBackend(o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache)
|
||||
lu, err = lookup.New(metadata.NewXattrsBackend(o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -136,7 +136,7 @@ func New(o *options.Options, stream events.Stream, cache, historyCache *idcache.
|
||||
|
||||
return filepath.Join(spaceRoot, lookup.MetadataDir)
|
||||
},
|
||||
o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache)
|
||||
o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+19
-10
@@ -39,6 +39,7 @@ import (
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
|
||||
@@ -453,9 +454,9 @@ func (t *Tree) assimilate(item scanItem) error {
|
||||
_ = unlock()
|
||||
}()
|
||||
|
||||
previousPath, ok := t.lookup.GetCachedID(context.Background(), spaceID, id)
|
||||
if previousPath == "" || !ok {
|
||||
previousPath, ok = t.lookup.IDHistoryCache.Get(context.Background(), spaceID, id)
|
||||
previousPath, err := t.lookup.GetCachedID(context.Background(), spaceID, id)
|
||||
if previousPath == "" || err != nil {
|
||||
previousPath, err = t.lookup.IDHistoryCache.Get(context.Background(), spaceID, id)
|
||||
}
|
||||
|
||||
// compare metadata mtime with actual mtime. if it matches AND the path hasn't changed (move operation)
|
||||
@@ -465,7 +466,7 @@ func (t *Tree) assimilate(item scanItem) error {
|
||||
}
|
||||
|
||||
// was it moved or copied/restored with a clashing id?
|
||||
if ok && len(parentID) > 0 && previousPath != item.Path {
|
||||
if err == nil && len(parentID) > 0 && previousPath != item.Path {
|
||||
_, err := os.Stat(previousPath)
|
||||
if err == nil {
|
||||
// this id clashes with an existing item -> clear metadata and re-assimilate
|
||||
@@ -942,14 +943,22 @@ func (t *Tree) WarmupIDCache(root string, assimilate, onlyDirty bool) error {
|
||||
if id != "" {
|
||||
// Check if the item on the previous path still exists. In this case it might have been a copy with extended attributes -> set new ID
|
||||
isCopy := false
|
||||
previousPath, ok := t.lookup.GetCachedID(context.Background(), spaceID, id)
|
||||
if ok && previousPath != path {
|
||||
_, err := os.Stat(previousPath)
|
||||
if err == nil {
|
||||
// previous path (using the same id) still exists -> this is a copy
|
||||
isCopy = true
|
||||
previousPath, err := t.lookup.GetCachedID(context.Background(), spaceID, id)
|
||||
switch err.(type) {
|
||||
case errtypes.NotFound:
|
||||
// previous path not found -> not a copy
|
||||
case nil:
|
||||
if previousPath != path {
|
||||
_, err := os.Stat(previousPath)
|
||||
if err == nil {
|
||||
// previous path (using the same id) still exists -> this is a copy
|
||||
isCopy = true
|
||||
}
|
||||
}
|
||||
default:
|
||||
return errors.Wrap(err, "failed to get previous path from cache")
|
||||
}
|
||||
|
||||
if isCopy {
|
||||
// copy detected -> re-assimilate
|
||||
_ = t.assimilate(scanItem{Path: path})
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
import re
|
||||
with open("/home/andre/src/opencloud/reva/pkg/storage/fs/posix/tree/assimilation_test.go", "r") as f:
|
||||
text = f.read()
|
||||
|
||||
new_content = """
|
||||
Describe("WarmupIDCache", func() {
|
||||
var (
|
||||
tree *Tree
|
||||
tmpDir string
|
||||
logger zerolog.Logger
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
tmpDir, err = os.MkdirTemp("", "warmupidcache-*")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
logger = zerolog.Nop()
|
||||
|
||||
tree = &Tree{
|
||||
log: &logger,
|
||||
options: &options.Options{
|
||||
Options: decomposedoptions.Options{
|
||||
Root: tmpDir,
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
os.RemoveAll(tmpDir)
|
||||
})
|
||||
|
||||
It("returns nil for an empty directory", func() {
|
||||
err := tree.WarmupIDCache(tmpDir, false, false)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
})
|
||||
"""
|
||||
|
||||
text = re.sub(r'Describe\("WarmupIDCache", func\(\).*$', new_content, text, flags=re.DOTALL)
|
||||
text += "\n})\n"
|
||||
with open("/home/andre/src/opencloud/reva/pkg/storage/fs/posix/tree/assimilation_test.go", "w") as f:
|
||||
f.write(text)
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
import re
|
||||
|
||||
with open("assimilation_test.go", "r") as f:
|
||||
text = f.read()
|
||||
|
||||
# remove CreateTestStorageSpace which throws error
|
||||
text = text.replace('_, err = env.CreateTestStorageSpace("personal", nil)\n\t\tExpect(err).ToNot(HaveOccurred())', '')
|
||||
|
||||
with open("assimilation_test.go", "w") as f:
|
||||
f.write(text)
|
||||
Generated
Vendored
+128
@@ -0,0 +1,128 @@
|
||||
import sys
|
||||
|
||||
content = """
|
||||
Describe("WarmupIDCache", func() {
|
||||
var (
|
||||
tree *Tree
|
||||
tmpDir string
|
||||
logger zerolog.Logger
|
||||
ctx context.Context
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
var err error
|
||||
tmpDir, err = os.MkdirTemp("", "warmupidcache-*")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
logger = zerolog.Nop()
|
||||
|
||||
o := &options.Options{
|
||||
Options: decomposedoptions.Options{
|
||||
Root: tmpDir,
|
||||
},
|
||||
}
|
||||
|
||||
// We need a backend and caches
|
||||
c, _ := idcache.NewMemoryIDCache()
|
||||
historyCache, _ := idcache.NewMemoryIDCache()
|
||||
um := &usermapper.NullMapper{}
|
||||
|
||||
backend := metadata.NewMessagePackBackend(o.FileMetadataCache)
|
||||
lu, err := lookup.New(backend, um, o, &timemanager.Manager{}, c, historyCache)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
tree = &Tree{
|
||||
log: &logger,
|
||||
options: o,
|
||||
lookup: lu,
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
os.RemoveAll(tmpDir)
|
||||
})
|
||||
|
||||
It("returns nil for an empty directory", func() {
|
||||
err := tree.WarmupIDCache(tmpDir, false, false)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("picks up new files and directories", func() {
|
||||
subDir := filepath.Join(tmpDir, "sub")
|
||||
err := os.Mkdir(subDir, 0755)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
filePath := filepath.Join(subDir, "test.txt")
|
||||
err = os.WriteFile(filePath, []byte("hello world"), 0644)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Should not crash, tests basic traverse
|
||||
err = tree.WarmupIDCache(tmpDir, false, false)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("verifies tree sizes and recursion", func() {
|
||||
// Setup a small directory structure
|
||||
subDir := filepath.Join(tmpDir, "sub2")
|
||||
err := os.Mkdir(subDir, 0755)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
nestedDir := filepath.Join(subDir, "nested")
|
||||
err = os.Mkdir(nestedDir, 0755)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
filePath := filepath.Join(nestedDir, "test.txt")
|
||||
err = os.WriteFile(filePath, []byte("hello world"), 0644) // 11 bytes
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Run assimilation
|
||||
err = tree.WarmupIDCache(tmpDir, true, false)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// If assimilation runs, the files will have xattrs and tree sizes evaluated
|
||||
// wait, but without mocked idResolver for the Tree, it might fail? Let's check!
|
||||
})
|
||||
})
|
||||
"""
|
||||
|
||||
with open("assimilation_test.go", "r") as f:
|
||||
text = f.read()
|
||||
|
||||
import re
|
||||
|
||||
# Add imports
|
||||
imports = """
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/idcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options"
|
||||
decomposedoptions "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
|
||||
)
|
||||
"""
|
||||
|
||||
# Replace imports
|
||||
start_import = text.find('import (')
|
||||
end_import = text.find(')', start_import) + 1
|
||||
text = text[:start_import] + imports.strip() + text[end_import:]
|
||||
|
||||
start_idx = text.find('Describe("WarmupIDCache", func() {')
|
||||
end_idx = text.find('})\n\n})', start_idx) + 2
|
||||
|
||||
if start_idx != -1 and end_idx != -1:
|
||||
new_text = text[:start_idx] + content.strip() + text[end_idx:]
|
||||
with open("assimilation_test.go", "w") as f:
|
||||
f.write(new_text)
|
||||
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
with open("assimilation_test.go", "r") as f:
|
||||
text = f.read()
|
||||
|
||||
import re
|
||||
|
||||
# We will inject the xattr check.
|
||||
insert = """// verify that tree sizes are updated
|
||||
// Since we used assimilate=true, the treesize xattr on sub2 and nested should be 11.
|
||||
|
||||
b, err := env.Lookup.MetadataBackend().Get(env.Ctx, subDir, "user.oc.treesize")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(string(b)).To(Equal("11"))
|
||||
|
||||
b, err = env.Lookup.MetadataBackend().Get(env.Ctx, nestedDir, "user.oc.treesize")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(string(b)).To(Equal("11"))"""
|
||||
|
||||
# inject right after env.Tree.WarmupIDCache call
|
||||
text = text.replace("err = env.Tree.WarmupIDCache(tmpDir+\"/users/admin\", true, false)\n\t\tExpect(err).ToNot(HaveOccurred())", "err = env.Tree.WarmupIDCache(tmpDir+\"/users/admin\", true, false)\n\t\tExpect(err).ToNot(HaveOccurred())\n\n" + insert)
|
||||
|
||||
with open("assimilation_test.go", "w") as f:
|
||||
f.write(text)
|
||||
|
||||
+28
-11
@@ -73,6 +73,10 @@ type Watcher interface {
|
||||
Watch(path string)
|
||||
}
|
||||
|
||||
type IDResolver interface {
|
||||
IDsForPath(ctx context.Context, path string) (spaceID string, nodeID string, err error)
|
||||
}
|
||||
|
||||
type scanItem struct {
|
||||
Path string
|
||||
Recurse bool
|
||||
@@ -80,12 +84,15 @@ type scanItem struct {
|
||||
|
||||
// Tree manages a hierarchical tree
|
||||
type Tree struct {
|
||||
lookup *lookup.Lookup
|
||||
blobstore node.Blobstore
|
||||
trashbin *trashbin.Trashbin
|
||||
propagator propagator.Propagator
|
||||
permissions permissions.Permissions
|
||||
|
||||
lookup *lookup.Lookup
|
||||
idResolver IDResolver // points at the lookup but can be overridden for testing
|
||||
assimilateFunc func(item scanItem) error // function to call to assimilate a node, can be overridden for testing
|
||||
|
||||
options *options.Options
|
||||
personalSpacesRoot string
|
||||
projectSpacesRoot string
|
||||
@@ -108,9 +115,10 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin *
|
||||
scanQueue := make(chan scanItem)
|
||||
|
||||
t := &Tree{
|
||||
lookup: lu.(*lookup.Lookup),
|
||||
blobstore: bs,
|
||||
userMapper: um,
|
||||
lookup: lu.(*lookup.Lookup),
|
||||
blobstore: bs,
|
||||
userMapper: um,
|
||||
// idResolver and assimilateFunc are wired below once t exists.
|
||||
trashbin: trashbin,
|
||||
permissions: permissions,
|
||||
options: o,
|
||||
@@ -125,6 +133,8 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin *
|
||||
personalSpacesRoot: filepath.Clean(filepath.Join(o.Root, templates.Base(o.PersonalSpacePathTemplate))),
|
||||
projectSpacesRoot: filepath.Clean(filepath.Join(o.Root, templates.Base(o.GeneralSpacePathTemplate))),
|
||||
}
|
||||
t.idResolver = t.lookup
|
||||
t.assimilateFunc = t.assimilate
|
||||
if err := t.checkStorage(); err != nil {
|
||||
return nil, errors.Wrap(err, "tree: unfit storage '"+o.Root+"'")
|
||||
}
|
||||
@@ -518,16 +528,23 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro
|
||||
for name := range work {
|
||||
path := filepath.Join(dir, name)
|
||||
|
||||
_, nodeID, err := t.lookup.IDsForPath(ctx, path)
|
||||
_, nodeID, err := t.idResolver.IDsForPath(ctx, path)
|
||||
if err != nil {
|
||||
// we don't know about this node yet for some reason, assimilate it on the fly
|
||||
if _, ok := err.(errtypes.IsNotFound); !ok {
|
||||
// a NotFound error just means we don't know about this
|
||||
// node yet. Any other error (e.g. an unavailable id
|
||||
// cache backend) is a real failure that must not be
|
||||
// silently turned into an assimilation attempt.
|
||||
return errors.Wrap(err, "tree: error resolving ids for "+path)
|
||||
}
|
||||
// we don't know about this node yet, assimilate it on the fly
|
||||
t.log.Info().Err(err).Str("path", path).Msg("encountered unknown entity while listing the directory. Assimilate.")
|
||||
err = t.assimilate(scanItem{Path: path})
|
||||
err = t.assimilateFunc(scanItem{Path: path})
|
||||
if err != nil {
|
||||
t.log.Error().Err(err).Str("path", path).Msg("failed to assimilate node")
|
||||
continue
|
||||
}
|
||||
_, nodeID, err = t.lookup.IDsForPath(ctx, path)
|
||||
_, nodeID, err = t.idResolver.IDsForPath(ctx, path)
|
||||
if err != nil || nodeID == "" {
|
||||
t.log.Error().Err(err).Str("path", path).Msg("still could not resolve node after assimilation")
|
||||
continue
|
||||
@@ -711,9 +728,9 @@ func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) {
|
||||
|
||||
idcache := t.lookup.IDCache
|
||||
// create a directory node
|
||||
parentPath, ok := idcache.Get(ctx, n.SpaceID, n.ParentID)
|
||||
if !ok {
|
||||
return errtypes.NotFound(n.ParentID)
|
||||
parentPath, err := idcache.Get(ctx, n.SpaceID, n.ParentID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path := filepath.Join(parentPath, n.Name)
|
||||
|
||||
|
||||
Generated
Vendored
+1
-1
@@ -1379,7 +1379,7 @@ func (fs *Decomposedfs) RestoreRecycleItem(ctx context.Context, space *provider.
|
||||
|
||||
// Warmup posix IDCache if restored path is a directory
|
||||
if cachingTree, ok := fs.tp.(IDCachingTree); ok {
|
||||
_ = cachingTree.WarmupIDCache(restoredNode.InternalPath(), false, false)
|
||||
_ = cachingTree.WarmupIDCache(restoredNode.InternalPath(), true, false)
|
||||
}
|
||||
} else {
|
||||
sizeDiff = restoredNode.Blobsize
|
||||
|
||||
+1
-1
@@ -173,7 +173,7 @@ type PathLookup interface {
|
||||
|
||||
type IDCacher interface {
|
||||
CacheID(ctx context.Context, spaceID, nodeID, val string) error
|
||||
GetCachedID(ctx context.Context, spaceID, nodeID string) (string, bool)
|
||||
GetCachedID(ctx context.Context, spaceID, nodeID string) (string, error)
|
||||
}
|
||||
|
||||
type BaseNode struct {
|
||||
|
||||
Vendored
+1
-1
@@ -1362,7 +1362,7 @@ github.com/opencloud-eu/icap-client
|
||||
# github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d
|
||||
## explicit; go 1.18
|
||||
github.com/opencloud-eu/libre-graph-api-go
|
||||
# github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721
|
||||
# github.com/opencloud-eu/reva/v2 v2.45.0
|
||||
## explicit; go 1.25.0
|
||||
github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace
|
||||
github.com/opencloud-eu/reva/v2/cmd/revad/runtime
|
||||
|
||||
Reference in New Issue
Block a user