diff --git a/services/search/pkg/opensearch/engine.go b/services/search/pkg/opensearch/engine.go index 09a5cf0f4..7a58c5ddc 100644 --- a/services/search/pkg/opensearch/engine.go +++ b/services/search/pkg/opensearch/engine.go @@ -24,9 +24,47 @@ type Engine struct { } func NewEngine(index string, client *opensearchgoAPI.Client) (*Engine, error) { - // first check if the cluster is healthy, we cannot expect that the index exists at this point, - // so we pass nil for the indices parameter and only check the cluster health - _, healthy, err := clusterHealth(context.Background(), client, nil) + pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{}) + switch { + case err != nil: + return nil, fmt.Errorf("%w, failed to ping opensearch: %w", ErrUnhealthyCluster, err) + case pingResp.IsError(): + return nil, fmt.Errorf("%w, failed to ping opensearch", ErrUnhealthyCluster) + } + + // apply the index template + if err := IndexTemplateResourceV1.Apply(context.TODO(), client); err != nil { + return nil, fmt.Errorf("failed to apply index template: %w", err) + } + + indicesExistsResp, err := client.Indices.Exists(context.TODO(), opensearchgoAPI.IndicesExistsReq{ + Indices: []string{index}, + }) + switch { + case indicesExistsResp != nil && indicesExistsResp.StatusCode == 404: + break + case err != nil: + return nil, fmt.Errorf("failed to check if index exists: %w", err) + case indicesExistsResp == nil: + return nil, fmt.Errorf("unexpected nil response when checking if index exists") + } + + // if the index does not exist, we need to create it + if indicesExistsResp.StatusCode == 404 { + resp, err := client.Indices.Create(context.TODO(), opensearchgoAPI.IndicesCreateReq{ + Index: index, + // the body is not necessary; we will use an index template to define the index settings and mappings + }) + switch { + case err != nil: + return nil, fmt.Errorf("failed to create index: %w", err) + case !resp.Acknowledged: + return nil, fmt.Errorf("failed to create index: %s", index) + } + } + + // first check if the cluster is healthy + _, healthy, err := clusterHealth(context.TODO(), client, []string{index}) switch { case err != nil: return nil, fmt.Errorf("failed to get cluster health: %w", err) @@ -34,12 +72,6 @@ func NewEngine(index string, client *opensearchgoAPI.Client) (*Engine, error) { return nil, fmt.Errorf("cluster health is not healthy") } - // apply the index template, this will create the index if it does not exist, - // or update it if it does exist - if err := IndexTemplateResourceV1.Apply(context.Background(), client); err != nil { - return nil, fmt.Errorf("failed to apply index template: %w", err) - } - return &Engine{index: index, client: client}, nil } @@ -49,12 +81,12 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque return nil, fmt.Errorf("failed to build query: %w", err) } - compiler, err := NewKQL() + transpiler, err := NewKQLToOsDSL() if err != nil { return nil, fmt.Errorf("failed to create KQL compiler: %w", err) } - builder, err := compiler.Compile(ast) + builder, err := transpiler.Compile(ast) if err != nil { return nil, fmt.Errorf("failed to compile query: %w", err) } @@ -124,7 +156,7 @@ func (e *Engine) Upsert(id string, r engine.Resource) error { return fmt.Errorf("failed to marshal resource: %w", err) } - _, err = e.client.Document.Create(context.Background(), opensearchgoAPI.DocumentCreateReq{ + _, err = e.client.Document.Create(context.TODO(), opensearchgoAPI.DocumentCreateReq{ Index: e.index, DocumentID: id, Body: bytes.NewReader(body), @@ -150,7 +182,7 @@ func (e *Engine) Delete(id string) error { return fmt.Errorf("failed to marshal body: %w", err) } - _, err = e.client.Update(context.Background(), opensearchgoAPI.UpdateReq{ + _, err = e.client.Update(context.TODO(), opensearchgoAPI.UpdateReq{ Index: e.index, DocumentID: id, Body: bytes.NewReader(body), @@ -172,7 +204,7 @@ func (e *Engine) Restore(id string) error { return fmt.Errorf("failed to marshal body: %w", err) } - _, err = e.client.Update(context.Background(), opensearchgoAPI.UpdateReq{ + _, err = e.client.Update(context.TODO(), opensearchgoAPI.UpdateReq{ Index: e.index, DocumentID: id, Body: bytes.NewReader(body), @@ -185,7 +217,7 @@ func (e *Engine) Restore(id string) error { } func (e *Engine) Purge(id string) error { - _, err := e.client.Document.Delete(context.Background(), opensearchgoAPI.DocumentDeleteReq{ + _, err := e.client.Document.Delete(context.TODO(), opensearchgoAPI.DocumentDeleteReq{ Index: e.index, DocumentID: id, }) @@ -204,7 +236,7 @@ func (e *Engine) DocCount() (uint64, error) { return 0, fmt.Errorf("failed to marshal query: %w", err) } - resp, err := e.client.Indices.Count(context.Background(), &opensearchgoAPI.IndicesCountReq{ + resp, err := e.client.Indices.Count(context.TODO(), &opensearchgoAPI.IndicesCountReq{ Indices: []string{e.index}, Body: bytes.NewReader(body), }) diff --git a/services/search/pkg/opensearch/export_test.go b/services/search/pkg/opensearch/export_test.go index c08ddc3c9..6e0228694 100644 --- a/services/search/pkg/opensearch/export_test.go +++ b/services/search/pkg/opensearch/export_test.go @@ -3,6 +3,7 @@ package opensearch var ( SearchHitToSearchMessageMatch = searchHitToSearchMessageMatch BuilderToBoolQuery = builderToBoolQuery + ExpandKQLASTNodes = expandKQLASTNodes ) func Convert[T any](v any) (T, error) { diff --git a/services/search/pkg/opensearch/internal/indices/resource_v1.json b/services/search/pkg/opensearch/internal/indices/resource_v1.json index 221fbd8e8..dd75bc8ef 100644 --- a/services/search/pkg/opensearch/internal/indices/resource_v1.json +++ b/services/search/pkg/opensearch/internal/indices/resource_v1.json @@ -1,6 +1,6 @@ { "index_patterns": [ - "opencloud-default-resource" + "opencloud-default-*" ], "template": { "settings": { @@ -18,6 +18,9 @@ "RootID": { "type": "keyword" }, + "MimeType": { + "type": "wildcard" + }, "Deleted": { "type": "boolean" }, diff --git a/services/search/pkg/opensearch/internal/test/test.go b/services/search/pkg/opensearch/internal/test/test.go index 6719b290f..b3eb83e02 100644 --- a/services/search/pkg/opensearch/internal/test/test.go +++ b/services/search/pkg/opensearch/internal/test/test.go @@ -5,4 +5,5 @@ type TableTest[G any, W any] struct { Got G Want W Err error + Skip bool } diff --git a/services/search/pkg/opensearch/kql.go b/services/search/pkg/opensearch/kql.go deleted file mode 100644 index ec77aa512..000000000 --- a/services/search/pkg/opensearch/kql.go +++ /dev/null @@ -1,174 +0,0 @@ -package opensearch - -import ( - "errors" - "fmt" - "strings" - "time" - - "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/pkg/kql" -) - -var ( - ErrUnsupportedNodeType = fmt.Errorf("unsupported node type") -) - -type KQL struct{} - -func NewKQL() (*KQL, error) { - return &KQL{}, nil -} - -func (k *KQL) Compile(tree *ast.Ast) (Builder, error) { - q, err := k.compile(tree.Nodes) - if err != nil { - return nil, err - } - - return q, nil -} - -func (k *KQL) compile(nodes []ast.Node) (Builder, error) { - if len(nodes) == 0 { - return nil, fmt.Errorf("no nodes to compile") - } - - if len(nodes) == 1 { - builder, err := k.getBuilder(nodes[0]) - if err != nil { - return nil, fmt.Errorf("failed to get builder for single node: %w", err) - } - return builder, nil - } - - boolQuery := NewBoolQuery() - add := boolQuery.Must - - for i, node := range nodes { - nextOp := k.getOperatorValueAt(nodes, i+1) - prevOp := k.getOperatorValueAt(nodes, i-1) - - switch { - case nextOp == kql.BoolOR: - add = boolQuery.Should - case nextOp == kql.BoolAND: - add = boolQuery.Must - case prevOp == kql.BoolNOT: - add = boolQuery.MustNot - } - - builder, err := k.getBuilder(node) - switch { - // if the node is not known, we skip it, such as an operator node - case errors.Is(err, ErrUnsupportedNodeType): - continue - case err != nil: - return nil, fmt.Errorf("failed to get builder for node %T: %w", node, err) - } - - if _, ok := node.(*ast.OperatorNode); ok { - // operatorNodes are not builders, so we skip them - continue - } - - add(builder) - } - - if len(boolQuery.should) != 0 { - boolQuery.options.MinimumShouldMatch = 1 - } - - return boolQuery, nil -} - -func (k *KQL) getFieldName(name string) string { - if name == "" { - return "Name" - } - - var _fields = map[string]string{ - "rootid": "RootID", - "path": "Path", - "id": "ID", - "name": "Name", - "size": "Size", - "mtime": "Mtime", - "mediatype": "MimeType", - "type": "Type", - "tag": "Tags", - "tags": "Tags", - "content": "Content", - "hidden": "Hidden", - } - - switch n, ok := _fields[strings.ToLower(name)]; { - case ok: - return n - default: - return name - } -} - -func (k *KQL) getOperatorValueAt(nodes []ast.Node, i int) string { - if i < 0 || i >= len(nodes) { - return "" - } - - if opn, ok := nodes[i].(*ast.OperatorNode); ok { - return opn.Value - } - - return "" -} - -func (k *KQL) getBuilder(node ast.Node) (Builder, error) { - var builder Builder - switch node := node.(type) { - case *ast.BooleanNode: - builder = NewTermQuery[bool](k.getFieldName(node.Key)).Value(node.Value) - case *ast.StringNode: - if strings.Contains(node.Value, "*") { - builder = NewWildcardQuery(k.getFieldName(node.Key)).Value(node.Value) - break - } - - switch len(strings.Split(node.Value, " ")) { - case 1: - builder = NewTermQuery[string](k.getFieldName(node.Key)).Value(node.Value) - default: - builder = NewMatchPhraseQuery(k.getFieldName(node.Key)).Query(node.Value) - } - case *ast.DateTimeNode: - if node.Operator == nil { - return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType) - } - - q := NewRangeQuery[time.Time](k.getFieldName(node.Key)) - - switch node.Operator.Value { - case ">": - q.Gt(node.Value) - case ">=": - q.Gte(node.Value) - case "<": - q.Lt(node.Value) - case "<=": - q.Lte(node.Value) - default: - return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType) - } - - return q, nil - case *ast.GroupNode: - group, err := k.compile(node.Nodes) - if err != nil { - return nil, fmt.Errorf("failed to build group: %w", err) - } - builder = group - default: - return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node) - } - - return builder, nil -} diff --git a/services/search/pkg/opensearch/kql_ast.go b/services/search/pkg/opensearch/kql_ast.go new file mode 100644 index 000000000..672379b4e --- /dev/null +++ b/services/search/pkg/opensearch/kql_ast.go @@ -0,0 +1,196 @@ +package opensearch + +import ( + "fmt" + "reflect" + "slices" + "strings" + + "github.com/opencloud-eu/opencloud/pkg/ast" +) + +func expandKQLASTNodes(nodes []ast.Node) ([]ast.Node, error) { + remapKey := func(current string, defaultKey string) string { + if defaultKey == "" { + defaultKey = "Name" // Set a default key if none is provided + } + + key, ok := map[string]string{ + "": defaultKey, // Default case if current is empty + "rootid": "RootID", + "path": "Path", + "id": "ID", + "name": "Name", + "size": "Size", + "mtime": "Mtime", + "mediatype": "MimeType", + "type": "Type", + "tag": "Tags", + "tags": "Tags", + "content": "Content", + "hidden": "Hidden", + }[current] + if !ok { + return current // Return the original key if not found + } + + return key + } + + lowerValue := func(key, value string) string { + if slices.Contains([]string{"Hidden"}, key) { + return value // ignore certain keys and return the original value + } + + return strings.ToLower(value) + } + + unfoldValue := func(key, value string) []ast.Node { + result, ok := map[string][]ast.Node{ + "MimeType:file": { + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, + }, + "MimeType:folder": { + &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, + }, + "MimeType:document": { + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: key, Value: "application/msword"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "text/plain"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "text/markdown"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/rtf"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.apple.pages"}, + }}, + }, + "MimeType:spreadsheet": { + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: key, Value: "application/vnd.ms-excel"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "text/csv"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"}, + }}, + }, + "MimeType:presentation": { + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"}, + }}, + }, + "MimeType:pdf": { + &ast.StringNode{Key: key, Value: "application/pdf"}, + }, + "MimeType:image": { + &ast.StringNode{Key: key, Value: "image/*"}, + }, + "MimeType:video": { + &ast.StringNode{Key: key, Value: "video/*"}, + }, + "MimeType:audio": { + &ast.StringNode{Key: key, Value: "audio/*"}, + }, + "MimeType:archive": { + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: key, Value: "application/zip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/gzip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-gzip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-7z-compressed"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-rar-compressed"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-tar"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-bzip2"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-bzip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-tgz"}, + }}, + }, + }[fmt.Sprintf("%s:%s", key, value)] + if !ok { + return nil + } + + return result + } + + var expand func([]ast.Node, string) ([]ast.Node, error) + expand = func(nodes []ast.Node, defaultKey string) ([]ast.Node, error) { + for i, node := range nodes { + rnode := reflect.ValueOf(node) + + // we need to ensure that the node is a pointer to an ast.Node in every case + if rnode.Kind() != reflect.Ptr { + ptr := reflect.New(rnode.Type()) + ptr.Elem().Set(rnode) + rnode = ptr + cnode, ok := rnode.Interface().(ast.Node) + if !ok { + return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface()) + } + + node = cnode // Update the original node to the pointer + nodes[i] = node // Update the original slice with the pointer + } + + var unfoldedNodes []ast.Node + switch cnode := node.(type) { + case *ast.GroupNode: + if cnode.Key != "" { // group nodes should not get a default key + cnode.Key = remapKey(cnode.Key, defaultKey) + } + + groupNodes, err := expand(cnode.Nodes, cnode.Key) + if err != nil { + return nil, err + } + cnode.Nodes = groupNodes + case *ast.StringNode: + cnode.Key = remapKey(cnode.Key, defaultKey) + cnode.Value = lowerValue(cnode.Key, cnode.Value) + unfoldedNodes = unfoldValue(cnode.Key, cnode.Value) + case *ast.DateTimeNode: + cnode.Key = remapKey(cnode.Key, defaultKey) + case *ast.BooleanNode: + cnode.Key = remapKey(cnode.Key, defaultKey) + } + + if unfoldedNodes != nil { + // Insert unfolded nodes at the current index + nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...) + // Adjust index to account for new nodes + i += len(unfoldedNodes) - 1 + } + } + + return nodes, nil + } + + return expand(nodes, "") +} diff --git a/services/search/pkg/opensearch/kql_ast_test.go b/services/search/pkg/opensearch/kql_ast_test.go new file mode 100644 index 000000000..f742b44e9 --- /dev/null +++ b/services/search/pkg/opensearch/kql_ast_test.go @@ -0,0 +1,606 @@ +package opensearch_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" +) + +func TestExpandKQLASTNodes(t *testing.T) { + t.Run("always converts a value node to a pointer node", func(t *testing.T) { + tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ + { + Name: "ast.node.V -> ast.node.PTR", + Got: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "b"}, + ast.OperatorNode{Value: "AND"}, + &ast.DateTimeNode{Key: "c"}, + &ast.OperatorNode{Value: "OR"}, + ast.DateTimeNode{Key: "d"}, + ast.OperatorNode{Value: "OR"}, + &ast.BooleanNode{Key: "f"}, + &ast.OperatorNode{Value: "NOT"}, + ast.BooleanNode{Key: "g"}, + ast.OperatorNode{Value: "NOT"}, + &ast.GroupNode{Key: "h", Nodes: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.GroupNode{Key: "h", Nodes: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.GroupNode{Key: "h", Nodes: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "b"}, + }}, + }}, + }}, + ast.GroupNode{Key: "i", Nodes: []ast.Node{ + ast.StringNode{Key: "a"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "b"}, + ast.OperatorNode{Value: "OR"}, + ast.GroupNode{Key: "h", Nodes: []ast.Node{ + ast.StringNode{Key: "a"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "b"}, + ast.OperatorNode{Value: "OR"}, + ast.GroupNode{Key: "h", Nodes: []ast.Node{ + ast.StringNode{Key: "a"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "b"}, + }}, + }}, + }}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "b"}, + &ast.OperatorNode{Value: "AND"}, + &ast.DateTimeNode{Key: "c"}, + &ast.OperatorNode{Value: "OR"}, + &ast.DateTimeNode{Key: "d"}, + &ast.OperatorNode{Value: "OR"}, + &ast.BooleanNode{Key: "f"}, + &ast.OperatorNode{Value: "NOT"}, + &ast.BooleanNode{Key: "g"}, + &ast.OperatorNode{Value: "NOT"}, + &ast.GroupNode{Key: "h", Nodes: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.GroupNode{Key: "h", Nodes: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.GroupNode{Key: "h", Nodes: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "b"}, + }}, + }}, + }}, + &ast.GroupNode{Key: "i", Nodes: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.GroupNode{Key: "h", Nodes: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.GroupNode{Key: "h", Nodes: []ast.Node{ + &ast.StringNode{Key: "a"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "b"}, + }}, + }}, + }}, + }, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + result, err := opensearch.ExpandKQLASTNodes(test.Got) + require.NoError(t, err) + require.Equal(t, test.Want, result) + }) + } + }) + + t.Run("remaps some keys", func(t *testing.T) { + var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node] + + for k, v := range map[string]string{ + "": "Name", // Default to "Name" if no key is provided + "rootid": "RootID", + "path": "Path", + "id": "ID", + "name": "Name", + "size": "Size", + "mtime": "Mtime", + "mediatype": "MimeType", + "type": "Type", + "tag": "Tags", + "tags": "Tags", + "content": "Content", + "hidden": "Hidden", + "any": "any", // Example of an unknown key that should remain unchanged + } { + tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{ + Name: fmt.Sprintf("%s -> %s", k, v), + Got: []ast.Node{ + &ast.StringNode{Key: k}, + &ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: k}, + ast.OperatorNode{Value: "AND"}, + &ast.DateTimeNode{Key: k}, + &ast.OperatorNode{Value: "OR"}, + ast.DateTimeNode{Key: k}, + ast.OperatorNode{Value: "OR"}, + &ast.BooleanNode{Key: k}, + &ast.OperatorNode{Value: "NOT"}, + ast.BooleanNode{Key: k}, + ast.OperatorNode{Value: "NOT"}, + &ast.GroupNode{Key: k, Nodes: []ast.Node{ + &ast.StringNode{Key: k}, + &ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: k}, + &ast.OperatorNode{Value: "OR"}, + &ast.GroupNode{Key: k, Nodes: []ast.Node{ + &ast.StringNode{Key: k}, + &ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: k}, + &ast.OperatorNode{Value: "OR"}, + &ast.GroupNode{Key: k, Nodes: []ast.Node{ + &ast.StringNode{Key: k}, + &ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: k}, + }}, + }}, + }}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: v}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: v}, + &ast.OperatorNode{Value: "AND"}, + &ast.DateTimeNode{Key: v}, + &ast.OperatorNode{Value: "OR"}, + &ast.DateTimeNode{Key: v}, + &ast.OperatorNode{Value: "OR"}, + &ast.BooleanNode{Key: v}, + &ast.OperatorNode{Value: "NOT"}, + &ast.BooleanNode{Key: v}, + &ast.OperatorNode{Value: "NOT"}, + &ast.GroupNode{Key: func() string { + switch { + case k == "": + return k + default: + return v + } + }(), Nodes: []ast.Node{ + &ast.StringNode{Key: v}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: v}, + &ast.OperatorNode{Value: "OR"}, + &ast.GroupNode{Key: func() string { + switch { + case k == "": + return k + default: + return v + } + }(), Nodes: []ast.Node{ + &ast.StringNode{Key: v}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: v}, + &ast.OperatorNode{Value: "OR"}, + &ast.GroupNode{Key: func() string { + switch { + case k == "": + return k + default: + return v + } + }(), Nodes: []ast.Node{ + &ast.StringNode{Key: v}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: v}, + }}, + }}, + }}, + }, + }) + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + result, err := opensearch.ExpandKQLASTNodes(test.Got) + require.NoError(t, err) + require.Equal(t, test.Want, result) + }) + } + }) + + t.Run("lowercases some values", func(t *testing.T) { + tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ + { + Name: "!Hidden: StringNode -> stringnode", + Got: []ast.Node{ + ast.StringNode{Key: "aBc", Value: "StringNode"}, + ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + ast.StringNode{Key: "aBc", Value: "StringNode"}, + }}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "stringnode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "stringnode"}, + }}, + }, + }, + { + Name: "Hidden: StringNode -> StringNode", + Got: []ast.Node{ + ast.StringNode{Key: "Hidden", Value: "StringNode"}, + ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + ast.StringNode{Key: "Hidden", Value: "StringNode"}, + }}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "StringNode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "StringNode"}, + }}, + }, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + result, err := opensearch.ExpandKQLASTNodes(test.Got) + require.NoError(t, err) + require.Equal(t, test.Want, result) + }) + } + }) + + t.Run("unfolds some values", func(t *testing.T) { + tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ + { + Name: "MimeType:unknown", + Got: []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "unknown"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "unknown"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + { + Name: "MimeType:file", + Got: []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "file"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + { + Name: "MimeType:folder", + Got: []ast.Node{ + ast.BooleanNode{Key: "Deleted", Value: false}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "MimeType", Value: "folder"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.BooleanNode{Key: "Deleted", Value: false}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + { + Name: "MimeType:document", + Got: []ast.Node{ + ast.BooleanNode{Key: "Deleted", Value: false}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "MimeType", Value: "document"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.BooleanNode{Key: "Deleted", Value: false}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "application/msword"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "text/plain"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "text/markdown"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/rtf"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + { + Name: "MimeType:spreadsheet", + Got: []ast.Node{ + ast.BooleanNode{Key: "Deleted", Value: false}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "MimeType", Value: "spreadsheet"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.BooleanNode{Key: "Deleted", Value: false}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "text/csv"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + { + Name: "MimeType:presentation", + Got: []ast.Node{ + ast.BooleanNode{Key: "Deleted", Value: false}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "MimeType", Value: "presentation"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.BooleanNode{Key: "Deleted", Value: false}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + { + Name: "MimeType:pdf", + Got: []ast.Node{ + ast.BooleanNode{Key: "Deleted", Value: false}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "MimeType", Value: "pdf"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.BooleanNode{Key: "Deleted", Value: false}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "MimeType", Value: "application/pdf"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + { + Name: "MimeType:image", + Got: []ast.Node{ + ast.BooleanNode{Key: "Deleted", Value: false}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "MimeType", Value: "image"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.BooleanNode{Key: "Deleted", Value: false}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "MimeType", Value: "image/*"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + { + Name: "MimeType:video", + Got: []ast.Node{ + ast.BooleanNode{Key: "Deleted", Value: false}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "MimeType", Value: "video"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.BooleanNode{Key: "Deleted", Value: false}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "MimeType", Value: "video/*"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + { + Name: "MimeType:audio", + Got: []ast.Node{ + ast.BooleanNode{Key: "Deleted", Value: false}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "MimeType", Value: "audio"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.BooleanNode{Key: "Deleted", Value: false}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "MimeType", Value: "audio/*"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + { + Name: "MimeType:archive", + Got: []ast.Node{ + ast.BooleanNode{Key: "Deleted", Value: false}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Key: "MimeType", Value: "archive"}, + ast.OperatorNode{Value: "AND"}, + ast.StringNode{Value: "some-name"}, + }, + Want: []ast.Node{ + &ast.BooleanNode{Key: "Deleted", Value: false}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "application/zip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/gzip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/x-gzip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/x-tar"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/x-bzip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "MimeType", Value: "application/x-tgz"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: `some-name`}, + }, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + if test.Skip { + t.Skip("Skipping test due to known issue") + } + result, err := opensearch.ExpandKQLASTNodes(test.Got) + require.NoError(t, err) + require.EqualValues(t, test.Want, result) + }) + } + }) + + t.Run("different cases", func(t *testing.T) { + tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ + { + Name: "use the group node key as default key", + Got: []ast.Node{ + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Value: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "c", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Key: "a", Nodes: []ast.Node{ + &ast.StringNode{Value: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "c", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{ + &ast.StringNode{Value: "file"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "c", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: "mediatype", Value: "file"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "c", Value: "d"}, + }}, + }, + Want: []ast.Node{ + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "c", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Key: "a", Nodes: []ast.Node{ + &ast.StringNode{Key: "a", Value: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "c", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "c", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "c", Value: "d"}, + }}, + }, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + if test.Skip { + t.Skip("Skipping test due to known issue") + } + result, err := opensearch.ExpandKQLASTNodes(test.Got) + require.NoError(t, err) + require.EqualValues(t, test.Want, result) + }) + } + }) +} diff --git a/services/search/pkg/opensearch/kql_to_os_dsl.go b/services/search/pkg/opensearch/kql_to_os_dsl.go new file mode 100644 index 000000000..b299f21cd --- /dev/null +++ b/services/search/pkg/opensearch/kql_to_os_dsl.go @@ -0,0 +1,154 @@ +package opensearch + +import ( + "errors" + "fmt" + "strings" + "time" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/pkg/kql" +) + +var ( + ErrUnsupportedNodeType = fmt.Errorf("unsupported node type") +) + +type KQLToOsDSL struct{} + +func NewKQLToOsDSL() (*KQLToOsDSL, error) { + return &KQLToOsDSL{}, nil +} + +func (k *KQLToOsDSL) Compile(tree *ast.Ast) (Builder, error) { + q, err := k.transpile(tree.Nodes) + if err != nil { + return nil, err + } + + return q, nil +} + +func (k *KQLToOsDSL) transpile(nodes []ast.Node) (Builder, error) { + if len(nodes) == 0 { + return nil, fmt.Errorf("no nodes to compile") + } + + expandedNodes, err := expandKQLASTNodes(nodes) + if err != nil { + return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err) + } + + if len(expandedNodes) == 1 { + builder, err := k.toBuilder(expandedNodes[0]) + if err != nil { + return nil, fmt.Errorf("failed to get builder for single node: %w", err) + } + return builder, nil + } + + boolQuery := NewBoolQuery() + add := boolQuery.Must + + for i, node := range expandedNodes { + nextOp := k.getOperatorValueAt(expandedNodes, i+1) + prevOp := k.getOperatorValueAt(expandedNodes, i-1) + + switch { + case nextOp == kql.BoolOR: + add = boolQuery.Should + case nextOp == kql.BoolAND: + add = boolQuery.Must + case prevOp == kql.BoolNOT: + add = boolQuery.MustNot + } + + builder, err := k.toBuilder(node) + switch { + // if the node is not known, we skip it, such as an operator node + case errors.Is(err, ErrUnsupportedNodeType): + continue + case err != nil: + return nil, fmt.Errorf("failed to get builder for node %T: %w", node, err) + } + + if _, ok := node.(*ast.OperatorNode); ok { + // operatorNodes are not builders, so we skip them + continue + } + + add(builder) + } + + if len(boolQuery.should) != 0 { + boolQuery.options.MinimumShouldMatch = 1 + } + + return boolQuery, nil +} + +func (k *KQLToOsDSL) getOperatorValueAt(nodes []ast.Node, i int) string { + if i < 0 || i >= len(nodes) { + return "" + } + + if opn, ok := nodes[i].(*ast.OperatorNode); ok { + return opn.Value + } + + return "" +} + +func (k *KQLToOsDSL) toBuilder(node ast.Node) (Builder, error) { + var builder Builder + + switch node := node.(type) { + case *ast.BooleanNode: + return NewTermQuery[bool](node.Key).Value(node.Value), nil + case *ast.StringNode: + isWildcard := strings.Contains(node.Value, "*") + if isWildcard { + return NewWildcardQuery(node.Key).Value(node.Value), nil + } + + totalTerms := strings.Split(node.Value, " ") + isSingleTerm := len(totalTerms) == 1 + isMultiTerm := len(totalTerms) >= 1 + switch { + case isSingleTerm: + return NewTermQuery[string](node.Key).Value(node.Value), nil + case isMultiTerm: + return NewMatchPhraseQuery(node.Key).Query(node.Value), nil + } + + return nil, fmt.Errorf("unsupported string node value: %s", node.Value) + case *ast.DateTimeNode: + if node.Operator == nil { + return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType) + } + + query := NewRangeQuery[time.Time](node.Key) + + switch node.Operator.Value { + case ">": + return query.Gt(node.Value), nil + case ">=": + return query.Gte(node.Value), nil + case "<": + return query.Lt(node.Value), nil + case "<=": + return query.Lte(node.Value), nil + } + + return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType) + case *ast.GroupNode: + group, err := k.transpile(node.Nodes) + if err != nil { + return nil, fmt.Errorf("failed to build group: %w", err) + } + + return group, nil + } + + return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node) +} diff --git a/services/search/pkg/opensearch/kql_test.go b/services/search/pkg/opensearch/kql_to_os_dsl_test.go similarity index 91% rename from services/search/pkg/opensearch/kql_test.go rename to services/search/pkg/opensearch/kql_to_os_dsl_test.go index 6a2c89683..500d18300 100644 --- a/services/search/pkg/opensearch/kql_test.go +++ b/services/search/pkg/opensearch/kql_to_os_dsl_test.go @@ -21,16 +21,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Value: "openCloud"}, }, }, - Want: opensearch.NewTermQuery[string]("Name").Value("openCloud"), - }, - { - Name: "remaps known field names", - Got: &ast.Ast{ - Nodes: []ast.Node{ - &ast.StringNode{Key: "mediatype", Value: "application/gzip"}, - }, - }, - Want: opensearch.NewTermQuery[string]("MimeType").Value("application/gzip"), + Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"), }, // kql to os dsl - type tests { @@ -40,7 +31,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: opensearch.NewTermQuery[string]("Name").Value("openCloud"), + Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"), }, { Name: "term query - boolean node - true", @@ -67,7 +58,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open cloud"}, }, }, - Want: opensearch.NewMatchPhraseQuery("Name").Query("open cloud"), + Want: opensearch.NewMatchPhraseQuery("Name").Query(`open cloud`), }, { Name: "wildcard query - string node", @@ -164,7 +155,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "name", Value: "openCloud"}, }, }, - Want: opensearch.NewTermQuery[string]("Name").Value("openCloud"), + Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"), }, { Name: "[* *]", @@ -176,7 +167,7 @@ func TestKQL_Compile(t *testing.T) { }, Want: opensearch.NewBoolQuery(). Must( - opensearch.NewTermQuery[string]("Name").Value("openCloud"), + opensearch.NewTermQuery[string]("Name").Value("opencloud"), opensearch.NewTermQuery[string]("age").Value("32"), ), }, @@ -191,7 +182,7 @@ func TestKQL_Compile(t *testing.T) { }, Want: opensearch.NewBoolQuery(). Must( - opensearch.NewTermQuery[string]("Name").Value("openCloud"), + opensearch.NewTermQuery[string]("Name").Value("opencloud"), opensearch.NewTermQuery[string]("age").Value("32"), ), }, @@ -206,7 +197,7 @@ func TestKQL_Compile(t *testing.T) { }, Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Should( - opensearch.NewTermQuery[string]("Name").Value("openCloud"), + opensearch.NewTermQuery[string]("Name").Value("opencloud"), opensearch.NewTermQuery[string]("age").Value("32"), ), }, @@ -234,7 +225,7 @@ func TestKQL_Compile(t *testing.T) { }, Want: opensearch.NewBoolQuery(). Must( - opensearch.NewTermQuery[string]("Name").Value("openCloud"), + opensearch.NewTermQuery[string]("Name").Value("opencloud"), ). MustNot( opensearch.NewTermQuery[string]("age").Value("32"), @@ -253,7 +244,7 @@ func TestKQL_Compile(t *testing.T) { }, Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Should( - opensearch.NewTermQuery[string]("Name").Value("openCloud"), + opensearch.NewTermQuery[string]("Name").Value("opencloud"), opensearch.NewTermQuery[string]("age").Value("32"), opensearch.NewTermQuery[string]("age").Value("44"), ), @@ -376,10 +367,14 @@ func TestKQL_Compile(t *testing.T) { for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - compiler, err := opensearch.NewKQL() + if test.Skip { + t.Skip("skipping test: " + test.Name) + } + + transpiler, err := opensearch.NewKQLToOsDSL() assert.NoError(t, err) - dsl, err := compiler.Compile(test.Got) + dsl, err := transpiler.Compile(test.Got) assert.NoError(t, err) assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, dsl))