Merge pull request #6371 from aduffeck/improve-complex-search

Improve complex search

services/search/pkg/engine/bleve.go

@@ -6,6 +6,7 @@ import (
 	"math"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -143,6 +144,7 @@ func (b *Bleve) Search(_ context.Context, sir *searchService.SearchIndexRequest)
 	}
 
 	bleveReq := bleve.NewSearchRequest(q)
+	bleveReq.Highlight = bleve.NewHighlight()
 
 	switch {
 	case sir.PageSize == -1:
@@ -364,10 +366,27 @@ func formatQuery(q string) string {
 		cq = strings.ReplaceAll(cq, strings.ToLower(field)+":", field+":")
 	}
 
-	if strings.Contains(cq, ":") {
+	fieldRe := regexp.MustCompile(`\w+:[^ ]+`)
+	if fieldRe.MatchString(cq) {
+		parts := strings.Split(cq, " ")
+
+		cq = ""
+		for _, part := range parts {
+			fieldParts := strings.SplitN(part, ":", 2)
+			if len(fieldParts) > 1 {
+				value := fieldParts[1]
+				if value != "T" && value != "F" {
+					value = strings.ToLower(value) // do a lowercase query unless this is a boolean flag
+				}
+				cq += fieldParts[0] + ":" + value + " "
+			} else {
+				cq += part + " "
+			}
+		}
 		return cq // Sophisticated field based search
 	}
 
 	// this is a basic filename search
+	cq = strings.ReplaceAll(cq, ":", `\:`)
 	return "Name:*" + strings.ReplaceAll(strings.ToLower(cq), " ", `\ `) + "*"
 }

services/search/pkg/engine/bleve_test.go

@@ -215,14 +215,14 @@ var _ = Describe("Bleve", func() {
 				}
 			})
 
-			It("uses a lower-case index", func() {
+			It("does a case-insensitive search", func() {
 				parentResource.Document.Name = "foo.pdf"
 
 				err := eng.Upsert(parentResource.ID, parentResource)
 				Expect(err).ToNot(HaveOccurred())
 
 				assertDocCount(rootResource.ID, "Name:foo*", 1)
-				assertDocCount(rootResource.ID, "Name:Foo*", 0)
+				assertDocCount(rootResource.ID, "Name:Foo*", 1)
 			})
 
 			Context("and an additional file in a subdirectory", func() {

services/search/pkg/search/service.go

@@ -30,6 +30,7 @@ import (
 
 const (
 	_spaceStateTrashed = "trashed"
+	_slowQueryDuration = 500 * time.Millisecond
 )
 
 // Searcher is the interface to the SearchService
@@ -255,19 +256,26 @@ func (s *Service) searchIndex(ctx context.Context, req *searchsvc.SearchRequest,
 		permissions = space.GetRootInfo().GetPermissionSet()
 	}
 
-	res, err := s.engine.Search(ctx, &searchsvc.SearchIndexRequest{
+	searchRequest := &searchsvc.SearchIndexRequest{
 		Query: req.Query,
 		Ref: &searchmsg.Reference{
 			ResourceId: searchRootID,
 			Path:       mountpointPrefix,
 		},
 		PageSize: req.PageSize,
-	})
+	}
+	start := time.Now()
+	res, err := s.engine.Search(ctx, searchRequest)
+	duration := time.Since(start)
 	if err != nil {
-		s.logger.Error().Err(err).Str("space", space.Id.OpaqueId).Msg("failed to search the index")
+		s.logger.Error().Err(err).Str("duration", fmt.Sprint(duration)).Str("space", space.Id.OpaqueId).Msg("failed to search the index")
 		return nil, err
 	}
-	s.logger.Debug().Str("space", space.Id.OpaqueId).Int("hits", len(res.Matches)).Msg("space search done")
+	if duration > _slowQueryDuration {
+		s.logger.Info().Interface("searchRequest", searchRequest).Str("duration", fmt.Sprint(duration)).Str("space", space.Id.OpaqueId).Int("hits", len(res.Matches)).Msg("slow space search")
+	} else {
+		s.logger.Debug().Interface("searchRequest", searchRequest).Str("duration", fmt.Sprint(duration)).Str("space", space.Id.OpaqueId).Int("hits", len(res.Matches)).Msg("space search done")
+	}
 
 	for _, match := range res.Matches {
 		if mountpointPrefix != "" {