mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-05-12 14:30:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Mark items as deleted (recursively) but still keep them around

Browse Source
This commit is contained in:
André Duffeck
2022-04-29 09:00:14 +02:00
parent 68c5e2366c
commit 6bd67c0f27
9 changed files with 246 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/search/pkg/search/index/index.go
+69 -14
View File
@@ -20,6 +20,7 @@ package index
import (
"context"
"errors"
"strings"
"time"
@@ -34,15 +35,17 @@ import (
)
type indexDocument struct {
RootID string
Path string
ID string
ParentID string
RootID string
Path string
ID string
Name string
Size uint64
Mtime string
MimeType string
Type uint64
Deleted bool
}
// Index represents a bleve based search index
@@ -77,15 +80,56 @@ func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error
return i.bleveIndex.Index(idToBleveId(ri.Id), entity)
}
// Remove removes an entity from the index
func (i *Index) Remove(id *sprovider.ResourceId) error {
// Delete marks an entity from the index as delete (still keeping it around)
func (i *Index) Delete(id *sprovider.ResourceId) error {
return i.markAsDeleted(idToBleveId(id))
}
func (i *Index) markAsDeleted(id string) error {
req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{id}))
req.Fields = []string{"*"}
res, err := i.bleveIndex.Search(req)
if err != nil {
return err
}
if res.Hits.Len() == 0 {
return errors.New("entity not found")
}
entity := fieldsToEntity(res.Hits[0].Fields)
entity.Deleted = true
if entity.Type == uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) {
query := bleve.NewConjunctionQuery(
bleve.NewQueryStringQuery("RootID:"+entity.RootID),
bleve.NewQueryStringQuery("Path:"+entity.Path+"/*"),
)
bleveReq := bleve.NewSearchRequest(query)
bleveReq.Fields = []string{"*"}
res, err := i.bleveIndex.Search(bleveReq)
if err != nil {
return err
}
for _, h := range res.Hits {
i.markAsDeleted(h.ID)
}
}
return i.bleveIndex.Index(entity.ID, entity)
}
// Purge removes an entity from the index
func (i *Index) Purge(id *sprovider.ResourceId) error {
return i.bleveIndex.Delete(idToBleveId(id))
}
// Search searches the index according to the criteria specified in the given SearchIndexRequest
func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) {
deletedQuery := bleve.NewBoolFieldQuery(false)
deletedQuery.SetField("Deleted")
query := bleve.NewConjunctionQuery(
bleve.NewQueryStringQuery("Name:"+req.Query),
deletedQuery, // Skip documents that have been marked as deleted
bleve.NewQueryStringQuery("RootID:"+req.Ref.ResourceId.StorageId+"!"+req.Ref.ResourceId.OpaqueId), // Limit search to the space
bleve.NewQueryStringQuery("Path:"+req.Ref.Path+"*"), // Limit search to this directory in the space
)
@@ -122,10 +166,11 @@ func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocume
RootID: idToBleveId(ref.ResourceId),
Path: ref.Path,
ID: idToBleveId(ri.Id),
ParentID: idToBleveId(ri.ParentId),
Name: ri.Path,
Size: ri.Size,
MimeType: ri.MimeType,
Type: uint64(ri.Type),
Deleted: false,
}
if ri.Mtime != nil {
@@ -135,6 +180,21 @@ func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocume
return doc
}
func fieldsToEntity(fields map[string]interface{}) *indexDocument {
doc := &indexDocument{
RootID: fields["RootID"].(string),
Path: fields["Path"].(string),
ID: fields["ID"].(string),
Name: fields["Name"].(string),
Size: uint64(fields["Size"].(float64)),
Mtime: fields["Mtime"].(string),
MimeType: fields["MimeType"].(string),
Deleted: fields["Deleted"].(bool),
Type: uint64(fields["Type"].(float64)),
}
return doc
}
func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) {
rootIDParts := strings.SplitN(fields["RootID"].(string), "!", 2)
IDParts := strings.SplitN(fields["ID"].(string), "!", 2)
@@ -154,20 +214,15 @@ func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) {
},
Name: fields["Name"].(string),
Size: uint64(fields["Size"].(float64)),
Type: uint64(fields["Type"].(float64)),
MimeType: fields["MimeType"].(string),
Deleted: fields["Deleted"].(bool),
},
}
if mtime, err := time.Parse(time.RFC3339, fields["Mtime"].(string)); err == nil {
match.Entity.LastModifiedTime = &timestamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())}
}
if fields["ParentID"] != "" {
parentIDParts := strings.SplitN(fields["ParentID"].(string), "!", 2)
match.Entity.ParentId = &searchmsg.ResourceID{
StorageId: parentIDParts[0],
OpaqueId: parentIDParts[1],
}
}
return match, nil
}
extensions/search/pkg/search/index/index_test.go
+117 -33
View File
@@ -18,10 +18,63 @@ var _ = Describe("Index", func() {
var (
i *index.Index
bleveIndex bleve.Index
ref *sprovider.Reference
ri *sprovider.ResourceInfo
ctx context.Context
ctx context.Context
rootId = &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "rootopaqueid",
}
ref = &sprovider.Reference{
ResourceId: rootId,
Path: "./foo.pdf",
}
ri = &sprovider.ResourceInfo{
Id: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "opaqueid",
},
ParentId: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "someopaqueid",
},
Path: "foo.pdf",
Size: 12345,
Type: sprovider.ResourceType_RESOURCE_TYPE_FILE,
MimeType: "application/pdf",
Mtime: &typesv1beta1.Timestamp{Seconds: 4000},
}
parentRef = &sprovider.Reference{
ResourceId: rootId,
Path: "./sudbir",
}
parentRi = &sprovider.ResourceInfo{
Id: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "parentopaqueid",
},
Path: "subdir",
Size: 12345,
Type: sprovider.ResourceType_RESOURCE_TYPE_CONTAINER,
Mtime: &typesv1beta1.Timestamp{Seconds: 4000},
}
childRef = &sprovider.Reference{
ResourceId: rootId,
Path: "./sudbir/child.pdf",
}
childRi = &sprovider.ResourceInfo{
Id: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "childopaqueid",
},
ParentId: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "parentopaqueid",
},
Path: "subdir",
Size: 12345,
Type: sprovider.ResourceType_RESOURCE_TYPE_FILE,
Mtime: &typesv1beta1.Timestamp{Seconds: 4000},
}
)
BeforeEach(func() {
@@ -31,28 +84,6 @@ var _ = Describe("Index", func() {
i, err = index.New(bleveIndex)
Expect(err).ToNot(HaveOccurred())
ref = &sprovider.Reference{
ResourceId: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "rootopaqueid",
},
Path: "./foo.pdf",
}
ri = &sprovider.ResourceInfo{
Id: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "opaqueid",
},
ParentId: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "parentopaqueid",
},
Path: "foo.pdf",
Size: 12345,
MimeType: "application/pdf",
Mtime: &typesv1beta1.Timestamp{Seconds: 4000},
}
})
Describe("New", func() {
@@ -127,8 +158,9 @@ var _ = Describe("Index", func() {
Expect(match.Entity.Id.OpaqueId).To(Equal(ri.Id.OpaqueId))
Expect(match.Entity.Name).To(Equal(ri.Path))
Expect(match.Entity.Size).To(Equal(ri.Size))
Expect(match.Entity.Type).To(Equal(uint64(ri.Type)))
Expect(match.Entity.MimeType).To(Equal(ri.MimeType))
Expect(match.Entity.ParentId.OpaqueId).To(Equal(ri.ParentId.OpaqueId))
Expect(match.Entity.Deleted).To(BeFalse())
Expect(uint64(match.Entity.LastModifiedTime.AsTime().Unix())).To(Equal(ri.Mtime.Seconds))
})
@@ -247,16 +279,68 @@ var _ = Describe("Index", func() {
})
Describe("Remove", func() {
It("removes a resource from the index", func() {
err := i.Add(ref, ri)
It("marks a resource as deleted", func() {
err := i.Add(parentRef, parentRi)
Expect(err).ToNot(HaveOccurred())
count, _ := bleveIndex.DocCount()
Expect(count).To(Equal(uint64(1)))
res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{
Query: "subdir",
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: rootId.StorageId,
OpaqueId: rootId.OpaqueId,
},
},
})
Expect(err).ToNot(HaveOccurred())
Expect(len(res.Matches)).To(Equal(1))
err = i.Remove(ri.Id)
err = i.Delete(parentRi.Id)
Expect(err).ToNot(HaveOccurred())
count, _ = bleveIndex.DocCount()
Expect(count).To(Equal(uint64(0)))
res, err = i.Search(ctx, &searchsvc.SearchIndexRequest{
Query: "subdir",
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: rootId.StorageId,
OpaqueId: rootId.OpaqueId,
},
},
})
Expect(err).ToNot(HaveOccurred())
Expect(len(res.Matches)).To(Equal(0))
})
It("also marks child resources as deleted", func() {
err := i.Add(parentRef, parentRi)
Expect(err).ToNot(HaveOccurred())
err = i.Add(childRef, childRi)
Expect(err).ToNot(HaveOccurred())
res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{
Query: "subdir",
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: rootId.StorageId,
OpaqueId: rootId.OpaqueId,
},
},
})
Expect(err).ToNot(HaveOccurred())
Expect(len(res.Matches)).To(Equal(2))
err = i.Delete(parentRi.Id)
Expect(err).ToNot(HaveOccurred())
res, err = i.Search(ctx, &searchsvc.SearchIndexRequest{
Query: "subdir",
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: rootId.StorageId,
OpaqueId: rootId.OpaqueId,
},
},
})
Expect(err).ToNot(HaveOccurred())
Expect(len(res.Matches)).To(Equal(0))
})
})
})
extensions/search/pkg/search/mocks/IndexClient.go
+16 -2
View File
@@ -30,6 +30,20 @@ func (_m *IndexClient) Add(ref *providerv1beta1.Reference, ri *providerv1beta1.R
return r0
}
// Delete provides a mock function with given fields: ri
func (_m *IndexClient) Delete(ri *providerv1beta1.ResourceId) error {
ret := _m.Called(ri)
var r0 error
if rf, ok := ret.Get(0).(func(*providerv1beta1.ResourceId) error); ok {
r0 = rf(ri)
} else {
r0 = ret.Error(0)
}
return r0
}
// DocCount provides a mock function with given fields:
func (_m *IndexClient) DocCount() (uint64, error) {
ret := _m.Called()
@@ -51,8 +65,8 @@ func (_m *IndexClient) DocCount() (uint64, error) {
return r0, r1
}
// Remove provides a mock function with given fields: ri
func (_m *IndexClient) Remove(ri *providerv1beta1.ResourceId) error {
// Purge provides a mock function with given fields: ri
func (_m *IndexClient) Purge(ri *providerv1beta1.ResourceId) error {
ret := _m.Called(ri)
var r0 error
extensions/search/pkg/search/provider/searchprovider.go
+1 -1
View File
@@ -67,7 +67,7 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach
Id: e.Executant,
}
case events.ItemTrashed:
err := p.indexClient.Remove(e.Id)
err := p.indexClient.Delete(e.Id)
if err != nil {
p.logger.Error().Err(err).Interface("Id", e.Id).Msg("failed to remove item from index")
}
extensions/search/pkg/search/provider/searchprovider_test.go
+1 -1
View File
@@ -122,7 +122,7 @@ var _ = Describe("Searchprovider", func() {
gwClient.On("Stat", mock.Anything, mock.Anything).Return(&sprovider.StatResponse{
Status: status.NewNotFound(context.Background(), ""),
}, nil)
indexClient.On("Remove", mock.MatchedBy(func(id *sprovider.ResourceId) bool {
indexClient.On("Delete", mock.MatchedBy(func(id *sprovider.ResourceId) bool {
return id.OpaqueId == ri.Id.OpaqueId
})).Return(nil).Run(func(args mock.Arguments) {
called = true
extensions/search/pkg/search/search.go
+2 -1
View File
@@ -38,6 +38,7 @@ type ProviderClient interface {
type IndexClient interface {
Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error)
Add(ref *providerv1beta1.Reference, ri *providerv1beta1.ResourceInfo) error
Remove(ri *providerv1beta1.ResourceId) error
Delete(ri *providerv1beta1.ResourceId) error
Purge(ri *providerv1beta1.ResourceId) error
DocCount() (uint64, error)
}
protogen/gen/ocis/messages/search/v0/search.pb.go
+31 -12
View File
@@ -144,6 +144,8 @@ type Entity struct {
LastModifiedTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=last_modified_time,json=lastModifiedTime,proto3" json:"last_modified_time,omitempty"`
MimeType string `protobuf:"bytes,7,opt,name=mime_type,json=mimeType,proto3" json:"mime_type,omitempty"`
Permissions string `protobuf:"bytes,8,opt,name=permissions,proto3" json:"permissions,omitempty"`
Type uint64 `protobuf:"varint,9,opt,name=type,proto3" json:"type,omitempty"`
Deleted bool `protobuf:"varint,10,opt,name=deleted,proto3" json:"deleted,omitempty"`
}
func (x *Entity) Reset() {
@@ -234,6 +236,20 @@ func (x *Entity) GetPermissions() string {
return ""
}
func (x *Entity) GetType() uint64 {
if x != nil {
return x.Type
}
return 0
}
func (x *Entity) GetDeleted() bool {
if x != nil {
return x.Deleted
}
return false
}
type Match struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -311,7 +327,7 @@ var file_ocis_messages_search_v0_search_proto_rawDesc = []byte{
0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49,
0x44, 0x52, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a,
0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74,
0x68, 0x22, 0xb8, 0x02, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x03,
0x68, 0x22, 0xe6, 0x02, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x03,
0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x63, 0x69, 0x73,
0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x03, 0x72,
@@ -330,17 +346,20 @@ var file_ocis_messages_search_v0_search_proto_rawDesc = []byte{
0x09, 0x6d, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09,
0x52, 0x08, 0x6d, 0x69, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x65,
0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52,
0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x56, 0x0a, 0x05,
0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x37, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18,
0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73,
0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e,
0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x14,
0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73,
0x63, 0x6f, 0x72, 0x65, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73,
0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63,
0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72,
0x63, 0x68, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04,
0x74, 0x79, 0x70, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65,
0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28,
0x08, 0x52, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x56, 0x0a, 0x05, 0x4d, 0x61,
0x74, 0x63, 0x68, 0x12, 0x37, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61,
0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x45, 0x6e,
0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x14, 0x0a, 0x05,
0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, 0x63, 0x6f,
0x72, 0x65, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63, 0x69, 0x73,
0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
protogen/gen/ocis/services/search/v0/search.swagger.json
+7
View File
@@ -185,6 +185,13 @@
},
"permissions": {
"type": "string"
},
"type": {
"type": "string",
"format": "uint64"
},
"deleted": {
"type": "boolean"
}
}
},
protogen/proto/ocis/messages/search/v0/search.proto
+2
View File
@@ -25,6 +25,8 @@ message Entity {
google.protobuf.Timestamp last_modified_time = 6;
string mime_type = 7;
string permissions = 8;
uint64 type = 9;
bool deleted = 10;
}
message Match {