mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-01-02 02:11:18 -06:00
enhancement(search): implement kql ast expansion helper and remove similar parts from the to os dsl query transpiler
This commit is contained in:
@@ -24,9 +24,47 @@ type Engine struct {
|
||||
}
|
||||
|
||||
func NewEngine(index string, client *opensearchgoAPI.Client) (*Engine, error) {
|
||||
// first check if the cluster is healthy, we cannot expect that the index exists at this point,
|
||||
// so we pass nil for the indices parameter and only check the cluster health
|
||||
_, healthy, err := clusterHealth(context.Background(), client, nil)
|
||||
pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{})
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("%w, failed to ping opensearch: %w", ErrUnhealthyCluster, err)
|
||||
case pingResp.IsError():
|
||||
return nil, fmt.Errorf("%w, failed to ping opensearch", ErrUnhealthyCluster)
|
||||
}
|
||||
|
||||
// apply the index template
|
||||
if err := IndexTemplateResourceV1.Apply(context.TODO(), client); err != nil {
|
||||
return nil, fmt.Errorf("failed to apply index template: %w", err)
|
||||
}
|
||||
|
||||
indicesExistsResp, err := client.Indices.Exists(context.TODO(), opensearchgoAPI.IndicesExistsReq{
|
||||
Indices: []string{index},
|
||||
})
|
||||
switch {
|
||||
case indicesExistsResp != nil && indicesExistsResp.StatusCode == 404:
|
||||
break
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to check if index exists: %w", err)
|
||||
case indicesExistsResp == nil:
|
||||
return nil, fmt.Errorf("unexpected nil response when checking if index exists")
|
||||
}
|
||||
|
||||
// if the index does not exist, we need to create it
|
||||
if indicesExistsResp.StatusCode == 404 {
|
||||
resp, err := client.Indices.Create(context.TODO(), opensearchgoAPI.IndicesCreateReq{
|
||||
Index: index,
|
||||
// the body is not necessary; we will use an index template to define the index settings and mappings
|
||||
})
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to create index: %w", err)
|
||||
case !resp.Acknowledged:
|
||||
return nil, fmt.Errorf("failed to create index: %s", index)
|
||||
}
|
||||
}
|
||||
|
||||
// first check if the cluster is healthy
|
||||
_, healthy, err := clusterHealth(context.TODO(), client, []string{index})
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to get cluster health: %w", err)
|
||||
@@ -34,12 +72,6 @@ func NewEngine(index string, client *opensearchgoAPI.Client) (*Engine, error) {
|
||||
return nil, fmt.Errorf("cluster health is not healthy")
|
||||
}
|
||||
|
||||
// apply the index template, this will create the index if it does not exist,
|
||||
// or update it if it does exist
|
||||
if err := IndexTemplateResourceV1.Apply(context.Background(), client); err != nil {
|
||||
return nil, fmt.Errorf("failed to apply index template: %w", err)
|
||||
}
|
||||
|
||||
return &Engine{index: index, client: client}, nil
|
||||
}
|
||||
|
||||
@@ -49,12 +81,12 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque
|
||||
return nil, fmt.Errorf("failed to build query: %w", err)
|
||||
}
|
||||
|
||||
compiler, err := NewKQL()
|
||||
transpiler, err := NewKQLToOsDSL()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create KQL compiler: %w", err)
|
||||
}
|
||||
|
||||
builder, err := compiler.Compile(ast)
|
||||
builder, err := transpiler.Compile(ast)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile query: %w", err)
|
||||
}
|
||||
@@ -124,7 +156,7 @@ func (e *Engine) Upsert(id string, r engine.Resource) error {
|
||||
return fmt.Errorf("failed to marshal resource: %w", err)
|
||||
}
|
||||
|
||||
_, err = e.client.Document.Create(context.Background(), opensearchgoAPI.DocumentCreateReq{
|
||||
_, err = e.client.Document.Create(context.TODO(), opensearchgoAPI.DocumentCreateReq{
|
||||
Index: e.index,
|
||||
DocumentID: id,
|
||||
Body: bytes.NewReader(body),
|
||||
@@ -150,7 +182,7 @@ func (e *Engine) Delete(id string) error {
|
||||
return fmt.Errorf("failed to marshal body: %w", err)
|
||||
}
|
||||
|
||||
_, err = e.client.Update(context.Background(), opensearchgoAPI.UpdateReq{
|
||||
_, err = e.client.Update(context.TODO(), opensearchgoAPI.UpdateReq{
|
||||
Index: e.index,
|
||||
DocumentID: id,
|
||||
Body: bytes.NewReader(body),
|
||||
@@ -172,7 +204,7 @@ func (e *Engine) Restore(id string) error {
|
||||
return fmt.Errorf("failed to marshal body: %w", err)
|
||||
}
|
||||
|
||||
_, err = e.client.Update(context.Background(), opensearchgoAPI.UpdateReq{
|
||||
_, err = e.client.Update(context.TODO(), opensearchgoAPI.UpdateReq{
|
||||
Index: e.index,
|
||||
DocumentID: id,
|
||||
Body: bytes.NewReader(body),
|
||||
@@ -185,7 +217,7 @@ func (e *Engine) Restore(id string) error {
|
||||
}
|
||||
|
||||
func (e *Engine) Purge(id string) error {
|
||||
_, err := e.client.Document.Delete(context.Background(), opensearchgoAPI.DocumentDeleteReq{
|
||||
_, err := e.client.Document.Delete(context.TODO(), opensearchgoAPI.DocumentDeleteReq{
|
||||
Index: e.index,
|
||||
DocumentID: id,
|
||||
})
|
||||
@@ -204,7 +236,7 @@ func (e *Engine) DocCount() (uint64, error) {
|
||||
return 0, fmt.Errorf("failed to marshal query: %w", err)
|
||||
}
|
||||
|
||||
resp, err := e.client.Indices.Count(context.Background(), &opensearchgoAPI.IndicesCountReq{
|
||||
resp, err := e.client.Indices.Count(context.TODO(), &opensearchgoAPI.IndicesCountReq{
|
||||
Indices: []string{e.index},
|
||||
Body: bytes.NewReader(body),
|
||||
})
|
||||
|
||||
@@ -3,6 +3,7 @@ package opensearch
|
||||
var (
|
||||
SearchHitToSearchMessageMatch = searchHitToSearchMessageMatch
|
||||
BuilderToBoolQuery = builderToBoolQuery
|
||||
ExpandKQLASTNodes = expandKQLASTNodes
|
||||
)
|
||||
|
||||
func Convert[T any](v any) (T, error) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"index_patterns": [
|
||||
"opencloud-default-resource"
|
||||
"opencloud-default-*"
|
||||
],
|
||||
"template": {
|
||||
"settings": {
|
||||
@@ -18,6 +18,9 @@
|
||||
"RootID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"MimeType": {
|
||||
"type": "wildcard"
|
||||
},
|
||||
"Deleted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
||||
@@ -5,4 +5,5 @@ type TableTest[G any, W any] struct {
|
||||
Got G
|
||||
Want W
|
||||
Err error
|
||||
Skip bool
|
||||
}
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
package opensearch
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnsupportedNodeType = fmt.Errorf("unsupported node type")
|
||||
)
|
||||
|
||||
type KQL struct{}
|
||||
|
||||
func NewKQL() (*KQL, error) {
|
||||
return &KQL{}, nil
|
||||
}
|
||||
|
||||
func (k *KQL) Compile(tree *ast.Ast) (Builder, error) {
|
||||
q, err := k.compile(tree.Nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return q, nil
|
||||
}
|
||||
|
||||
func (k *KQL) compile(nodes []ast.Node) (Builder, error) {
|
||||
if len(nodes) == 0 {
|
||||
return nil, fmt.Errorf("no nodes to compile")
|
||||
}
|
||||
|
||||
if len(nodes) == 1 {
|
||||
builder, err := k.getBuilder(nodes[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get builder for single node: %w", err)
|
||||
}
|
||||
return builder, nil
|
||||
}
|
||||
|
||||
boolQuery := NewBoolQuery()
|
||||
add := boolQuery.Must
|
||||
|
||||
for i, node := range nodes {
|
||||
nextOp := k.getOperatorValueAt(nodes, i+1)
|
||||
prevOp := k.getOperatorValueAt(nodes, i-1)
|
||||
|
||||
switch {
|
||||
case nextOp == kql.BoolOR:
|
||||
add = boolQuery.Should
|
||||
case nextOp == kql.BoolAND:
|
||||
add = boolQuery.Must
|
||||
case prevOp == kql.BoolNOT:
|
||||
add = boolQuery.MustNot
|
||||
}
|
||||
|
||||
builder, err := k.getBuilder(node)
|
||||
switch {
|
||||
// if the node is not known, we skip it, such as an operator node
|
||||
case errors.Is(err, ErrUnsupportedNodeType):
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to get builder for node %T: %w", node, err)
|
||||
}
|
||||
|
||||
if _, ok := node.(*ast.OperatorNode); ok {
|
||||
// operatorNodes are not builders, so we skip them
|
||||
continue
|
||||
}
|
||||
|
||||
add(builder)
|
||||
}
|
||||
|
||||
if len(boolQuery.should) != 0 {
|
||||
boolQuery.options.MinimumShouldMatch = 1
|
||||
}
|
||||
|
||||
return boolQuery, nil
|
||||
}
|
||||
|
||||
func (k *KQL) getFieldName(name string) string {
|
||||
if name == "" {
|
||||
return "Name"
|
||||
}
|
||||
|
||||
var _fields = map[string]string{
|
||||
"rootid": "RootID",
|
||||
"path": "Path",
|
||||
"id": "ID",
|
||||
"name": "Name",
|
||||
"size": "Size",
|
||||
"mtime": "Mtime",
|
||||
"mediatype": "MimeType",
|
||||
"type": "Type",
|
||||
"tag": "Tags",
|
||||
"tags": "Tags",
|
||||
"content": "Content",
|
||||
"hidden": "Hidden",
|
||||
}
|
||||
|
||||
switch n, ok := _fields[strings.ToLower(name)]; {
|
||||
case ok:
|
||||
return n
|
||||
default:
|
||||
return name
|
||||
}
|
||||
}
|
||||
|
||||
func (k *KQL) getOperatorValueAt(nodes []ast.Node, i int) string {
|
||||
if i < 0 || i >= len(nodes) {
|
||||
return ""
|
||||
}
|
||||
|
||||
if opn, ok := nodes[i].(*ast.OperatorNode); ok {
|
||||
return opn.Value
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func (k *KQL) getBuilder(node ast.Node) (Builder, error) {
|
||||
var builder Builder
|
||||
switch node := node.(type) {
|
||||
case *ast.BooleanNode:
|
||||
builder = NewTermQuery[bool](k.getFieldName(node.Key)).Value(node.Value)
|
||||
case *ast.StringNode:
|
||||
if strings.Contains(node.Value, "*") {
|
||||
builder = NewWildcardQuery(k.getFieldName(node.Key)).Value(node.Value)
|
||||
break
|
||||
}
|
||||
|
||||
switch len(strings.Split(node.Value, " ")) {
|
||||
case 1:
|
||||
builder = NewTermQuery[string](k.getFieldName(node.Key)).Value(node.Value)
|
||||
default:
|
||||
builder = NewMatchPhraseQuery(k.getFieldName(node.Key)).Query(node.Value)
|
||||
}
|
||||
case *ast.DateTimeNode:
|
||||
if node.Operator == nil {
|
||||
return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
q := NewRangeQuery[time.Time](k.getFieldName(node.Key))
|
||||
|
||||
switch node.Operator.Value {
|
||||
case ">":
|
||||
q.Gt(node.Value)
|
||||
case ">=":
|
||||
q.Gte(node.Value)
|
||||
case "<":
|
||||
q.Lt(node.Value)
|
||||
case "<=":
|
||||
q.Lte(node.Value)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
return q, nil
|
||||
case *ast.GroupNode:
|
||||
group, err := k.compile(node.Nodes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build group: %w", err)
|
||||
}
|
||||
builder = group
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node)
|
||||
}
|
||||
|
||||
return builder, nil
|
||||
}
|
||||
196
services/search/pkg/opensearch/kql_ast.go
Normal file
196
services/search/pkg/opensearch/kql_ast.go
Normal file
@@ -0,0 +1,196 @@
|
||||
package opensearch
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
)
|
||||
|
||||
func expandKQLASTNodes(nodes []ast.Node) ([]ast.Node, error) {
|
||||
remapKey := func(current string, defaultKey string) string {
|
||||
if defaultKey == "" {
|
||||
defaultKey = "Name" // Set a default key if none is provided
|
||||
}
|
||||
|
||||
key, ok := map[string]string{
|
||||
"": defaultKey, // Default case if current is empty
|
||||
"rootid": "RootID",
|
||||
"path": "Path",
|
||||
"id": "ID",
|
||||
"name": "Name",
|
||||
"size": "Size",
|
||||
"mtime": "Mtime",
|
||||
"mediatype": "MimeType",
|
||||
"type": "Type",
|
||||
"tag": "Tags",
|
||||
"tags": "Tags",
|
||||
"content": "Content",
|
||||
"hidden": "Hidden",
|
||||
}[current]
|
||||
if !ok {
|
||||
return current // Return the original key if not found
|
||||
}
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
lowerValue := func(key, value string) string {
|
||||
if slices.Contains([]string{"Hidden"}, key) {
|
||||
return value // ignore certain keys and return the original value
|
||||
}
|
||||
|
||||
return strings.ToLower(value)
|
||||
}
|
||||
|
||||
unfoldValue := func(key, value string) []ast.Node {
|
||||
result, ok := map[string][]ast.Node{
|
||||
"MimeType:file": {
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
|
||||
},
|
||||
"MimeType:folder": {
|
||||
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
|
||||
},
|
||||
"MimeType:document": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/msword"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "text/plain"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "text/markdown"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/rtf"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.apple.pages"},
|
||||
}},
|
||||
},
|
||||
"MimeType:spreadsheet": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.ms-excel"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "text/csv"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"},
|
||||
}},
|
||||
},
|
||||
"MimeType:presentation": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"},
|
||||
}},
|
||||
},
|
||||
"MimeType:pdf": {
|
||||
&ast.StringNode{Key: key, Value: "application/pdf"},
|
||||
},
|
||||
"MimeType:image": {
|
||||
&ast.StringNode{Key: key, Value: "image/*"},
|
||||
},
|
||||
"MimeType:video": {
|
||||
&ast.StringNode{Key: key, Value: "video/*"},
|
||||
},
|
||||
"MimeType:audio": {
|
||||
&ast.StringNode{Key: key, Value: "audio/*"},
|
||||
},
|
||||
"MimeType:archive": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/zip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-7z-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-rar-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-tar"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-bzip2"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-bzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-tgz"},
|
||||
}},
|
||||
},
|
||||
}[fmt.Sprintf("%s:%s", key, value)]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
var expand func([]ast.Node, string) ([]ast.Node, error)
|
||||
expand = func(nodes []ast.Node, defaultKey string) ([]ast.Node, error) {
|
||||
for i, node := range nodes {
|
||||
rnode := reflect.ValueOf(node)
|
||||
|
||||
// we need to ensure that the node is a pointer to an ast.Node in every case
|
||||
if rnode.Kind() != reflect.Ptr {
|
||||
ptr := reflect.New(rnode.Type())
|
||||
ptr.Elem().Set(rnode)
|
||||
rnode = ptr
|
||||
cnode, ok := rnode.Interface().(ast.Node)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface())
|
||||
}
|
||||
|
||||
node = cnode // Update the original node to the pointer
|
||||
nodes[i] = node // Update the original slice with the pointer
|
||||
}
|
||||
|
||||
var unfoldedNodes []ast.Node
|
||||
switch cnode := node.(type) {
|
||||
case *ast.GroupNode:
|
||||
if cnode.Key != "" { // group nodes should not get a default key
|
||||
cnode.Key = remapKey(cnode.Key, defaultKey)
|
||||
}
|
||||
|
||||
groupNodes, err := expand(cnode.Nodes, cnode.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cnode.Nodes = groupNodes
|
||||
case *ast.StringNode:
|
||||
cnode.Key = remapKey(cnode.Key, defaultKey)
|
||||
cnode.Value = lowerValue(cnode.Key, cnode.Value)
|
||||
unfoldedNodes = unfoldValue(cnode.Key, cnode.Value)
|
||||
case *ast.DateTimeNode:
|
||||
cnode.Key = remapKey(cnode.Key, defaultKey)
|
||||
case *ast.BooleanNode:
|
||||
cnode.Key = remapKey(cnode.Key, defaultKey)
|
||||
}
|
||||
|
||||
if unfoldedNodes != nil {
|
||||
// Insert unfolded nodes at the current index
|
||||
nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...)
|
||||
// Adjust index to account for new nodes
|
||||
i += len(unfoldedNodes) - 1
|
||||
}
|
||||
}
|
||||
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
return expand(nodes, "")
|
||||
}
|
||||
606
services/search/pkg/opensearch/kql_ast_test.go
Normal file
606
services/search/pkg/opensearch/kql_ast_test.go
Normal file
@@ -0,0 +1,606 @@
|
||||
package opensearch_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
|
||||
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestExpandKQLASTNodes(t *testing.T) {
|
||||
t.Run("always converts a value node to a pointer node", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "ast.node.V -> ast.node.PTR",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: "c"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
ast.DateTimeNode{Key: "d"},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: "f"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
ast.BooleanNode{Key: "g"},
|
||||
ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
ast.GroupNode{Key: "i", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "a"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "a"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "a"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: "c"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.DateTimeNode{Key: "d"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: "f"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.BooleanNode{Key: "g"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
&ast.GroupNode{Key: "i", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
result, err := opensearch.ExpandKQLASTNodes(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("remaps some keys", func(t *testing.T) {
|
||||
var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node]
|
||||
|
||||
for k, v := range map[string]string{
|
||||
"": "Name", // Default to "Name" if no key is provided
|
||||
"rootid": "RootID",
|
||||
"path": "Path",
|
||||
"id": "ID",
|
||||
"name": "Name",
|
||||
"size": "Size",
|
||||
"mtime": "Mtime",
|
||||
"mediatype": "MimeType",
|
||||
"type": "Type",
|
||||
"tag": "Tags",
|
||||
"tags": "Tags",
|
||||
"content": "Content",
|
||||
"hidden": "Hidden",
|
||||
"any": "any", // Example of an unknown key that should remain unchanged
|
||||
} {
|
||||
tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
Name: fmt.Sprintf("%s -> %s", k, v),
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: k},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
ast.DateTimeNode{Key: k},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: k},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
ast.BooleanNode{Key: k},
|
||||
ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: k, Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: k, Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: k, Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.DateTimeNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: v},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.BooleanNode{Key: v},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: func() string {
|
||||
switch {
|
||||
case k == "":
|
||||
return k
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}(), Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: func() string {
|
||||
switch {
|
||||
case k == "":
|
||||
return k
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}(), Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: func() string {
|
||||
switch {
|
||||
case k == "":
|
||||
return k
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}(), Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
result, err := opensearch.ExpandKQLASTNodes(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("lowercases some values", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "!Hidden: StringNode -> stringnode",
|
||||
Got: []ast.Node{
|
||||
ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "aBc", Value: "stringnode"},
|
||||
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "aBc", Value: "stringnode"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Hidden: StringNode -> StringNode",
|
||||
Got: []ast.Node{
|
||||
ast.StringNode{Key: "Hidden", Value: "StringNode"},
|
||||
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "Hidden", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "Hidden", Value: "StringNode"},
|
||||
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Hidden", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
result, err := opensearch.ExpandKQLASTNodes(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unfolds some values", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "MimeType:unknown",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "unknown"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "unknown"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:file",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "file"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:folder",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "folder"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:document",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "document"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/msword"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "text/plain"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "text/markdown"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/rtf"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:spreadsheet",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "spreadsheet"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "text/csv"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:presentation",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "presentation"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:pdf",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "pdf"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/pdf"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:image",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "image"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "image/*"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:video",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "video"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "video/*"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:audio",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "audio"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "audio/*"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:archive",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "archive"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/zip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-tar"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-tgz"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
if test.Skip {
|
||||
t.Skip("Skipping test due to known issue")
|
||||
}
|
||||
result, err := opensearch.ExpandKQLASTNodes(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("different cases", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "use the group node key as default key",
|
||||
Got: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "file"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "mediatype", Value: "file"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
if test.Skip {
|
||||
t.Skip("Skipping test due to known issue")
|
||||
}
|
||||
result, err := opensearch.ExpandKQLASTNodes(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
154
services/search/pkg/opensearch/kql_to_os_dsl.go
Normal file
154
services/search/pkg/opensearch/kql_to_os_dsl.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package opensearch
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnsupportedNodeType = fmt.Errorf("unsupported node type")
|
||||
)
|
||||
|
||||
type KQLToOsDSL struct{}
|
||||
|
||||
func NewKQLToOsDSL() (*KQLToOsDSL, error) {
|
||||
return &KQLToOsDSL{}, nil
|
||||
}
|
||||
|
||||
func (k *KQLToOsDSL) Compile(tree *ast.Ast) (Builder, error) {
|
||||
q, err := k.transpile(tree.Nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return q, nil
|
||||
}
|
||||
|
||||
func (k *KQLToOsDSL) transpile(nodes []ast.Node) (Builder, error) {
|
||||
if len(nodes) == 0 {
|
||||
return nil, fmt.Errorf("no nodes to compile")
|
||||
}
|
||||
|
||||
expandedNodes, err := expandKQLASTNodes(nodes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err)
|
||||
}
|
||||
|
||||
if len(expandedNodes) == 1 {
|
||||
builder, err := k.toBuilder(expandedNodes[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get builder for single node: %w", err)
|
||||
}
|
||||
return builder, nil
|
||||
}
|
||||
|
||||
boolQuery := NewBoolQuery()
|
||||
add := boolQuery.Must
|
||||
|
||||
for i, node := range expandedNodes {
|
||||
nextOp := k.getOperatorValueAt(expandedNodes, i+1)
|
||||
prevOp := k.getOperatorValueAt(expandedNodes, i-1)
|
||||
|
||||
switch {
|
||||
case nextOp == kql.BoolOR:
|
||||
add = boolQuery.Should
|
||||
case nextOp == kql.BoolAND:
|
||||
add = boolQuery.Must
|
||||
case prevOp == kql.BoolNOT:
|
||||
add = boolQuery.MustNot
|
||||
}
|
||||
|
||||
builder, err := k.toBuilder(node)
|
||||
switch {
|
||||
// if the node is not known, we skip it, such as an operator node
|
||||
case errors.Is(err, ErrUnsupportedNodeType):
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to get builder for node %T: %w", node, err)
|
||||
}
|
||||
|
||||
if _, ok := node.(*ast.OperatorNode); ok {
|
||||
// operatorNodes are not builders, so we skip them
|
||||
continue
|
||||
}
|
||||
|
||||
add(builder)
|
||||
}
|
||||
|
||||
if len(boolQuery.should) != 0 {
|
||||
boolQuery.options.MinimumShouldMatch = 1
|
||||
}
|
||||
|
||||
return boolQuery, nil
|
||||
}
|
||||
|
||||
func (k *KQLToOsDSL) getOperatorValueAt(nodes []ast.Node, i int) string {
|
||||
if i < 0 || i >= len(nodes) {
|
||||
return ""
|
||||
}
|
||||
|
||||
if opn, ok := nodes[i].(*ast.OperatorNode); ok {
|
||||
return opn.Value
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func (k *KQLToOsDSL) toBuilder(node ast.Node) (Builder, error) {
|
||||
var builder Builder
|
||||
|
||||
switch node := node.(type) {
|
||||
case *ast.BooleanNode:
|
||||
return NewTermQuery[bool](node.Key).Value(node.Value), nil
|
||||
case *ast.StringNode:
|
||||
isWildcard := strings.Contains(node.Value, "*")
|
||||
if isWildcard {
|
||||
return NewWildcardQuery(node.Key).Value(node.Value), nil
|
||||
}
|
||||
|
||||
totalTerms := strings.Split(node.Value, " ")
|
||||
isSingleTerm := len(totalTerms) == 1
|
||||
isMultiTerm := len(totalTerms) >= 1
|
||||
switch {
|
||||
case isSingleTerm:
|
||||
return NewTermQuery[string](node.Key).Value(node.Value), nil
|
||||
case isMultiTerm:
|
||||
return NewMatchPhraseQuery(node.Key).Query(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported string node value: %s", node.Value)
|
||||
case *ast.DateTimeNode:
|
||||
if node.Operator == nil {
|
||||
return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
query := NewRangeQuery[time.Time](node.Key)
|
||||
|
||||
switch node.Operator.Value {
|
||||
case ">":
|
||||
return query.Gt(node.Value), nil
|
||||
case ">=":
|
||||
return query.Gte(node.Value), nil
|
||||
case "<":
|
||||
return query.Lt(node.Value), nil
|
||||
case "<=":
|
||||
return query.Lte(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
|
||||
case *ast.GroupNode:
|
||||
group, err := k.transpile(node.Nodes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build group: %w", err)
|
||||
}
|
||||
|
||||
return group, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node)
|
||||
}
|
||||
@@ -21,16 +21,7 @@ func TestKQL_Compile(t *testing.T) {
|
||||
&ast.StringNode{Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: opensearch.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
},
|
||||
{
|
||||
Name: "remaps known field names",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "mediatype", Value: "application/gzip"},
|
||||
},
|
||||
},
|
||||
Want: opensearch.NewTermQuery[string]("MimeType").Value("application/gzip"),
|
||||
Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"),
|
||||
},
|
||||
// kql to os dsl - type tests
|
||||
{
|
||||
@@ -40,7 +31,7 @@ func TestKQL_Compile(t *testing.T) {
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: opensearch.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"),
|
||||
},
|
||||
{
|
||||
Name: "term query - boolean node - true",
|
||||
@@ -67,7 +58,7 @@ func TestKQL_Compile(t *testing.T) {
|
||||
&ast.StringNode{Key: "Name", Value: "open cloud"},
|
||||
},
|
||||
},
|
||||
Want: opensearch.NewMatchPhraseQuery("Name").Query("open cloud"),
|
||||
Want: opensearch.NewMatchPhraseQuery("Name").Query(`open cloud`),
|
||||
},
|
||||
{
|
||||
Name: "wildcard query - string node",
|
||||
@@ -164,7 +155,7 @@ func TestKQL_Compile(t *testing.T) {
|
||||
&ast.StringNode{Key: "name", Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: opensearch.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"),
|
||||
},
|
||||
{
|
||||
Name: "[* *]",
|
||||
@@ -176,7 +167,7 @@ func TestKQL_Compile(t *testing.T) {
|
||||
},
|
||||
Want: opensearch.NewBoolQuery().
|
||||
Must(
|
||||
opensearch.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
opensearch.NewTermQuery[string]("Name").Value("opencloud"),
|
||||
opensearch.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -191,7 +182,7 @@ func TestKQL_Compile(t *testing.T) {
|
||||
},
|
||||
Want: opensearch.NewBoolQuery().
|
||||
Must(
|
||||
opensearch.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
opensearch.NewTermQuery[string]("Name").Value("opencloud"),
|
||||
opensearch.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -206,7 +197,7 @@ func TestKQL_Compile(t *testing.T) {
|
||||
},
|
||||
Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
opensearch.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
opensearch.NewTermQuery[string]("Name").Value("opencloud"),
|
||||
opensearch.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -234,7 +225,7 @@ func TestKQL_Compile(t *testing.T) {
|
||||
},
|
||||
Want: opensearch.NewBoolQuery().
|
||||
Must(
|
||||
opensearch.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
opensearch.NewTermQuery[string]("Name").Value("opencloud"),
|
||||
).
|
||||
MustNot(
|
||||
opensearch.NewTermQuery[string]("age").Value("32"),
|
||||
@@ -253,7 +244,7 @@ func TestKQL_Compile(t *testing.T) {
|
||||
},
|
||||
Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
opensearch.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
opensearch.NewTermQuery[string]("Name").Value("opencloud"),
|
||||
opensearch.NewTermQuery[string]("age").Value("32"),
|
||||
opensearch.NewTermQuery[string]("age").Value("44"),
|
||||
),
|
||||
@@ -376,10 +367,14 @@ func TestKQL_Compile(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
compiler, err := opensearch.NewKQL()
|
||||
if test.Skip {
|
||||
t.Skip("skipping test: " + test.Name)
|
||||
}
|
||||
|
||||
transpiler, err := opensearch.NewKQLToOsDSL()
|
||||
assert.NoError(t, err)
|
||||
|
||||
dsl, err := compiler.Compile(test.Got)
|
||||
dsl, err := transpiler.Compile(test.Got)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, dsl))
|
||||
Reference in New Issue
Block a user