From 1586f7fcbb1e42ed0b5b890f2e4196c5dee10e63 Mon Sep 17 00:00:00 2001 From: fschade Date: Wed, 6 Aug 2025 14:43:58 +0200 Subject: [PATCH] enhancement(search): implement index manager and remove the use of index templates --- go.mod | 3 +- go.sum | 1 + services/search/pkg/opensearch/engine.go | 30 +- services/search/pkg/opensearch/engine_test.go | 126 +-- .../internal/indexes/resource_v1.json | 33 + .../internal/indices/resource_v1.json | 40 - .../pkg/opensearch/os_dsl_query_root.go | 2 +- .../pkg/opensearch/os_dsl_query_root_test.go | 2 +- services/search/pkg/opensearch/os_index.go | 142 ++++ .../pkg/opensearch/os_index_template.go | 65 -- .../pkg/opensearch/os_index_template_test.go | 34 - .../search/pkg/opensearch/os_index_test.go | 62 ++ vendor/github.com/tidwall/sjson/LICENSE | 21 + vendor/github.com/tidwall/sjson/README.md | 278 +++++++ vendor/github.com/tidwall/sjson/logo.png | Bin 0 -> 16874 bytes vendor/github.com/tidwall/sjson/sjson.go | 737 ++++++++++++++++++ vendor/modules.txt | 3 + 17 files changed, 1346 insertions(+), 233 deletions(-) create mode 100644 services/search/pkg/opensearch/internal/indexes/resource_v1.json delete mode 100644 services/search/pkg/opensearch/internal/indices/resource_v1.json create mode 100644 services/search/pkg/opensearch/os_index.go delete mode 100644 services/search/pkg/opensearch/os_index_template.go delete mode 100644 services/search/pkg/opensearch/os_index_template_test.go create mode 100644 services/search/pkg/opensearch/os_index_test.go create mode 100644 vendor/github.com/tidwall/sjson/LICENSE create mode 100644 vendor/github.com/tidwall/sjson/README.md create mode 100644 vendor/github.com/tidwall/sjson/logo.png create mode 100644 vendor/github.com/tidwall/sjson/sjson.go diff --git a/go.mod b/go.mod index 491bef2b6..934a106d3 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/ggwhite/go-masker v1.1.0 github.com/go-chi/chi/v5 v5.2.2 github.com/go-chi/render v1.0.3 + github.com/go-jose/go-jose/v3 v3.0.4 github.com/go-ldap/ldap/v3 v3.4.11 github.com/go-ldap/ldif v0.0.0-20200320164324-fd88d9b715b3 github.com/go-micro/plugins/v4/client/grpc v1.2.1 @@ -84,6 +85,7 @@ require ( github.com/theckman/yacspin v0.13.12 github.com/thejerf/suture/v4 v4.0.6 github.com/tidwall/gjson v1.18.0 + github.com/tidwall/sjson v1.2.5 github.com/tus/tusd/v2 v2.8.0 github.com/unrolled/secure v1.16.0 github.com/urfave/cli/v2 v2.27.7 @@ -188,7 +190,6 @@ require ( github.com/go-git/go-billy/v5 v5.6.2 // indirect github.com/go-git/go-git/v5 v5.13.2 // indirect github.com/go-ini/ini v1.67.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.4 // indirect github.com/go-jose/go-jose/v4 v4.1.1 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect diff --git a/go.sum b/go.sum index 3d9251e79..04c1f3619 100644 --- a/go.sum +++ b/go.sum @@ -1087,6 +1087,7 @@ github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRz github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg= github.com/thejerf/suture/v4 v4.0.6 h1:QsuCEsCqb03xF9tPAsWAj8QOAJBgQI1c0VqJNaingg8= github.com/thejerf/suture/v4 v4.0.6/go.mod h1:gu9Y4dXNUWFrByqRt30Rm9/UZ0wzRSt9AJS6xu/ZGxU= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= diff --git a/services/search/pkg/opensearch/engine.go b/services/search/pkg/opensearch/engine.go index 0a58b4a33..63b04ddf2 100644 --- a/services/search/pkg/opensearch/engine.go +++ b/services/search/pkg/opensearch/engine.go @@ -39,36 +39,10 @@ func NewEngine(index string, client *opensearchgoAPI.Client) (*Engine, error) { } // apply the index template - if err := IndexTemplateResourceV1.Apply(context.TODO(), client); err != nil { + if err := IndexManagerLatest.Apply(context.TODO(), index, client); err != nil { return nil, fmt.Errorf("failed to apply index template: %w", err) } - indicesExistsResp, err := client.Indices.Exists(context.TODO(), opensearchgoAPI.IndicesExistsReq{ - Indices: []string{index}, - }) - switch { - case indicesExistsResp != nil && indicesExistsResp.StatusCode == 404: - break - case err != nil: - return nil, fmt.Errorf("failed to check if index exists: %w", err) - case indicesExistsResp == nil: - return nil, fmt.Errorf("unexpected nil response when checking if index exists") - } - - // if the index does not exist, we need to create it - if indicesExistsResp.StatusCode == 404 { - resp, err := client.Indices.Create(context.TODO(), opensearchgoAPI.IndicesCreateReq{ - Index: index, - // the body is not necessary; we will use an index template to define the index settings and mappings - }) - switch { - case err != nil: - return nil, fmt.Errorf("failed to create index: %w", err) - case !resp.Acknowledged: - return nil, fmt.Errorf("failed to create index: %s", index) - } - } - // first check if the cluster is healthy _, healthy, err := clusterHealth(context.TODO(), client, []string{index}) switch { @@ -116,7 +90,7 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque } body, err := NewRootQuery(boolQuery, RootQueryOptions{ - Highlight: RootQueryHighlight{ + Highlight: &RootQueryHighlight{ PreTags: []string{""}, PostTags: []string{""}, Fields: map[string]RootQueryHighlight{ diff --git a/services/search/pkg/opensearch/engine_test.go b/services/search/pkg/opensearch/engine_test.go index 953bf8bea..e57c41a04 100644 --- a/services/search/pkg/opensearch/engine_test.go +++ b/services/search/pkg/opensearch/engine_test.go @@ -23,26 +23,26 @@ func TestNewEngine(t *testing.T) { }) require.NoError(t, err, "failed to create OpenSearch client") - engine, err := opensearch.NewEngine("test-engine-new-engine", client) - require.Nil(t, engine) + backend, err := opensearch.NewEngine("test-engine-new-engine", client) + require.Nil(t, backend) require.ErrorIs(t, err, opensearch.ErrUnhealthyCluster) }) } func TestEngine_Search(t *testing.T) { - index := "opencloud-default-resource" + indexName := "opencloud-test-resource" tc := opensearchtest.NewDefaultTestClient(t) - tc.Require.IndicesReset([]string{index}) - tc.Require.IndicesCount([]string{index}, "", 0) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, "", 0) - defer tc.Require.IndicesDelete([]string{index}) + defer tc.Require.IndicesDelete([]string{indexName}) + + backend, err := opensearch.NewEngine(indexName, tc.Client()) + require.NoError(t, err) document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) - tc.Require.IndicesCount([]string{index}, "", 1) - - backend, err := opensearch.NewEngine(index, tc.Client()) - require.NoError(t, err) + tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document)) + tc.Require.IndicesCount([]string{indexName}, "", 1) t.Run("most simple search", func(t *testing.T) { resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{ @@ -59,8 +59,8 @@ func TestEngine_Search(t *testing.T) { deletedDocument.ID = "1$2!4" deletedDocument.Deleted = true - tc.Require.DocumentCreate(index, deletedDocument.ID, opensearchtest.JSONMustMarshal(t, deletedDocument)) - tc.Require.IndicesCount([]string{index}, "", 2) + tc.Require.DocumentCreate(indexName, deletedDocument.ID, opensearchtest.JSONMustMarshal(t, deletedDocument)) + tc.Require.IndicesCount([]string{indexName}, "", 2) resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{ Query: fmt.Sprintf(`"%s"`, document.Name), @@ -73,43 +73,43 @@ func TestEngine_Search(t *testing.T) { } func TestEngine_Upsert(t *testing.T) { - index := "opencloud-default-resource" + indexName := "opencloud-test-resource" tc := opensearchtest.NewDefaultTestClient(t) - tc.Require.IndicesReset([]string{index}) - tc.Require.IndicesCount([]string{index}, "", 0) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, "", 0) - defer tc.Require.IndicesDelete([]string{index}) + defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(index, tc.Client()) + backend, err := opensearch.NewEngine(indexName, tc.Client()) require.NoError(t, err) t.Run("upsert with full document", func(t *testing.T) { document := opensearchtest.Testdata.Resources.File require.NoError(t, backend.Upsert(document.ID, document)) - tc.Require.IndicesCount([]string{index}, "", 1) + tc.Require.IndicesCount([]string{indexName}, "", 1) }) } func TestEngine_Move(t *testing.T) { - index := "opencloud-default-resource" + indexName := "opencloud-test-resource" tc := opensearchtest.NewDefaultTestClient(t) - tc.Require.IndicesReset([]string{index}) - tc.Require.IndicesCount([]string{index}, "", 0) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, "", 0) - defer tc.Require.IndicesDelete([]string{index}) + defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(index, tc.Client()) + backend, err := opensearch.NewEngine(indexName, tc.Client()) require.NoError(t, err) t.Run("moves the document to a new path", func(t *testing.T) { document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) - tc.Require.IndicesCount([]string{index}, "", 1) + tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document)) + tc.Require.IndicesCount([]string{indexName}, "", 1) resources := opensearchtest.SearchHitsMustBeConverted[engine.Resource](t, tc.Require.Search( - index, + indexName, opensearch.NewRootQuery( opensearch.NewIDsQuery([]string{document.ID}), ).String(), @@ -123,7 +123,7 @@ func TestEngine_Move(t *testing.T) { resources = opensearchtest.SearchHitsMustBeConverted[engine.Resource](t, tc.Require.Search( - index, + indexName, opensearch.NewRootQuery( opensearch.NewIDsQuery([]string{document.ID}), ).String(), @@ -135,109 +135,109 @@ func TestEngine_Move(t *testing.T) { } func TestEngine_Delete(t *testing.T) { - index := "opencloud-default-resource" + indexName := "opencloud-test-resource" tc := opensearchtest.NewDefaultTestClient(t) - tc.Require.IndicesReset([]string{index}) - tc.Require.IndicesCount([]string{index}, "", 0) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, "", 0) - defer tc.Require.IndicesDelete([]string{index}) + defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(index, tc.Client()) + backend, err := opensearch.NewEngine(indexName, tc.Client()) require.NoError(t, err) t.Run("mark document as deleted", func(t *testing.T) { document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) - tc.Require.IndicesCount([]string{index}, "", 1) + tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document)) + tc.Require.IndicesCount([]string{indexName}, "", 1) - tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery( + tc.Require.IndicesCount([]string{indexName}, opensearch.NewRootQuery( opensearch.NewTermQuery[bool]("Deleted").Value(true), ).String(), 0) require.NoError(t, backend.Delete(document.ID)) - tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery( + tc.Require.IndicesCount([]string{indexName}, opensearch.NewRootQuery( opensearch.NewTermQuery[bool]("Deleted").Value(true), ).String(), 1) }) } func TestEngine_Restore(t *testing.T) { - index := "opencloud-default-resource" + indexName := "opencloud-test-resource" tc := opensearchtest.NewDefaultTestClient(t) - tc.Require.IndicesReset([]string{index}) - tc.Require.IndicesCount([]string{index}, "", 0) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, "", 0) - defer tc.Require.IndicesDelete([]string{index}) + defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(index, tc.Client()) + backend, err := opensearch.NewEngine(indexName, tc.Client()) require.NoError(t, err) t.Run("mark document as not deleted", func(t *testing.T) { document := opensearchtest.Testdata.Resources.File document.Deleted = true - tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) - tc.Require.IndicesCount([]string{index}, "", 1) + tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document)) + tc.Require.IndicesCount([]string{indexName}, "", 1) - tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery( + tc.Require.IndicesCount([]string{indexName}, opensearch.NewRootQuery( opensearch.NewTermQuery[bool]("Deleted").Value(true), ).String(), 1) require.NoError(t, backend.Restore(document.ID)) - tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery( + tc.Require.IndicesCount([]string{indexName}, opensearch.NewRootQuery( opensearch.NewTermQuery[bool]("Deleted").Value(true), ).String(), 0) }) } func TestEngine_Purge(t *testing.T) { - index := "opencloud-default-resource" + indexName := "opencloud-test-resource" tc := opensearchtest.NewDefaultTestClient(t) - tc.Require.IndicesReset([]string{index}) - tc.Require.IndicesCount([]string{index}, "", 0) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, "", 0) - defer tc.Require.IndicesDelete([]string{index}) + defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(index, tc.Client()) + backend, err := opensearch.NewEngine(indexName, tc.Client()) require.NoError(t, err) t.Run("purge with full document", func(t *testing.T) { document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) - tc.Require.IndicesCount([]string{index}, "", 1) + tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document)) + tc.Require.IndicesCount([]string{indexName}, "", 1) require.NoError(t, backend.Purge(document.ID)) - tc.Require.IndicesCount([]string{index}, "", 0) + tc.Require.IndicesCount([]string{indexName}, "", 0) }) } func TestEngine_DocCount(t *testing.T) { - index := "opencloud-default-resource" + indexName := "opencloud-test-resource" tc := opensearchtest.NewDefaultTestClient(t) - tc.Require.IndicesReset([]string{index}) - tc.Require.IndicesCount([]string{index}, "", 0) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, "", 0) - defer tc.Require.IndicesDelete([]string{index}) + defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(index, tc.Client()) + backend, err := opensearch.NewEngine(indexName, tc.Client()) require.NoError(t, err) t.Run("ignore deleted documents", func(t *testing.T) { document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) - tc.Require.IndicesCount([]string{index}, "", 1) + tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document)) + tc.Require.IndicesCount([]string{indexName}, "", 1) count, err := backend.DocCount() require.NoError(t, err) require.Equal(t, uint64(1), count) - tc.Require.Update(index, document.ID, opensearchtest.JSONMustMarshal(t, map[string]any{ + tc.Require.Update(indexName, document.ID, opensearchtest.JSONMustMarshal(t, map[string]any{ "doc": map[string]any{ "Deleted": true, }, })) - tc.Require.IndicesCount([]string{index}, "", 1) + tc.Require.IndicesCount([]string{indexName}, "", 1) count, err = backend.DocCount() require.NoError(t, err) diff --git a/services/search/pkg/opensearch/internal/indexes/resource_v1.json b/services/search/pkg/opensearch/internal/indexes/resource_v1.json new file mode 100644 index 000000000..d2adc5807 --- /dev/null +++ b/services/search/pkg/opensearch/internal/indexes/resource_v1.json @@ -0,0 +1,33 @@ +{ + "settings": { + "number_of_shards": "1", + "number_of_replicas": "1" + }, + "mappings": { + "properties": { + "ID": { + "type": "keyword" + }, + "ParentID": { + "type": "keyword" + }, + "RootID": { + "type": "keyword" + }, + "MimeType": { + "type": "wildcard", + "doc_values": false + }, + "Path": { + "type": "wildcard", + "doc_values": false + }, + "Deleted": { + "type": "boolean" + }, + "Hidden": { + "type": "boolean" + } + } + } +} diff --git a/services/search/pkg/opensearch/internal/indices/resource_v1.json b/services/search/pkg/opensearch/internal/indices/resource_v1.json deleted file mode 100644 index 8852dfb18..000000000 --- a/services/search/pkg/opensearch/internal/indices/resource_v1.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "index_patterns": [ - "opencloud-default-*" - ], - "template": { - "settings": { - "number_of_shards": 1, - "number_of_replicas": 1 - }, - "mappings": { - "properties": { - "ID": { - "type": "keyword" - }, - "ParentID": { - "type": "keyword" - }, - "RootID": { - "type": "keyword" - }, - "MimeType": { - "type": "wildcard" - }, - "Path": { - "type": "wildcard" - }, - "Deleted": { - "type": "boolean" - }, - "Hidden": { - "type": "boolean" - } - } - } - }, - "version": 1, - "_meta": { - "description": "using component templates" - } -} diff --git a/services/search/pkg/opensearch/os_dsl_query_root.go b/services/search/pkg/opensearch/os_dsl_query_root.go index 1dd6805eb..f02087b92 100644 --- a/services/search/pkg/opensearch/os_dsl_query_root.go +++ b/services/search/pkg/opensearch/os_dsl_query_root.go @@ -50,7 +50,7 @@ func (q *RootQuery) String() string { } type RootQueryOptions struct { - Highlight RootQueryHighlight `json:"highlight,omitempty"` + Highlight *RootQueryHighlight `json:"highlight,omitempty"` } type RootQueryHighlight struct { diff --git a/services/search/pkg/opensearch/os_dsl_query_root_test.go b/services/search/pkg/opensearch/os_dsl_query_root_test.go index d47efa9c4..c88f7c186 100644 --- a/services/search/pkg/opensearch/os_dsl_query_root_test.go +++ b/services/search/pkg/opensearch/os_dsl_query_root_test.go @@ -29,7 +29,7 @@ func TestRootQuery(t *testing.T) { Got: opensearch.NewRootQuery( opensearch.NewTermQuery[string]("content").Value("content"), opensearch.RootQueryOptions{ - Highlight: opensearch.RootQueryHighlight{ + Highlight: &opensearch.RootQueryHighlight{ PreTags: []string{""}, PostTags: []string{""}, Fields: map[string]opensearch.RootQueryHighlight{ diff --git a/services/search/pkg/opensearch/os_index.go b/services/search/pkg/opensearch/os_index.go new file mode 100644 index 000000000..ef69cf904 --- /dev/null +++ b/services/search/pkg/opensearch/os_index.go @@ -0,0 +1,142 @@ +package opensearch + +import ( + "bytes" + "context" + "embed" + "errors" + "fmt" + "path" + "reflect" + + "github.com/go-jose/go-jose/v3/json" + opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" + "github.com/tidwall/gjson" +) + +var ( + ErrManualActionRequired = errors.New("manual action required") + IndexManagerLatest = IndexIndexManagerResourceV1 + IndexIndexManagerResourceV1 IndexManager = "resource_v1.json" +) + +//go:embed internal/indexes/*.json +var indexes embed.FS + +type IndexManager string + +func (m IndexManager) String() string { + b, err := m.MarshalJSON() + if err != nil { + return "" + } + + return string(b) +} + +func (m IndexManager) MarshalJSON() ([]byte, error) { + filePath := string(m) + body, err := indexes.ReadFile(path.Join("./internal/indexes", filePath)) + switch { + case err != nil: + return nil, fmt.Errorf("failed to read index file %s: %w", filePath, err) + case len(body) <= 0: + return nil, fmt.Errorf("index file %s is empty", filePath) + } + + return body, nil +} + +func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client) error { + localIndexB, err := m.MarshalJSON() + if err != nil { + return fmt.Errorf("failed to marshal index %s: %w", name, err) + } + + indicesExistsResp, err := client.Indices.Exists(ctx, opensearchgoAPI.IndicesExistsReq{ + Indices: []string{name}, + }) + switch { + case indicesExistsResp != nil && indicesExistsResp.StatusCode == 404: + break + case err != nil: + return fmt.Errorf("failed to check if index %s exists: %w", name, err) + case indicesExistsResp == nil: + return fmt.Errorf("indicesExistsResp is nil for index %s", name) + } + + if indicesExistsResp.StatusCode == 200 { + resp, err := client.Indices.Get(ctx, opensearchgoAPI.IndicesGetReq{ + Indices: []string{name}, + }) + if err != nil { + return fmt.Errorf("failed to get index %s: %w", name, err) + } + + remoteIndex, ok := resp.Indices[name] + if !ok { + return fmt.Errorf("index %s not found in response", name) + } + remoteIndexB, err := json.Marshal(remoteIndex) + if err != nil { + return fmt.Errorf("failed to marshal index %s: %w", name, err) + } + + localIndexJson := gjson.ParseBytes(localIndexB) + remoteIndexJson := gjson.ParseBytes(remoteIndexB) + + compare := func(lvPath, rvPath string) (any, any, bool) { + lv := localIndexJson.Get(lvPath).Raw + rv := remoteIndexJson.Get(rvPath).Raw + + var lvv, rvv interface{} + if err := json.Unmarshal([]byte(lv), &lvv); err != nil { + return nil, nil, false + } + + if err := json.Unmarshal([]byte(rv), &rvv); err != nil { + return nil, nil, false + } + + return lv, rv, reflect.DeepEqual(lvv, rvv) + } + + var errs []error + + for k := range localIndexJson.Get("settings").Map() { + if lv, rv, ok := compare("settings."+k, "settings.index."+k); !ok { + errs = append(errs, fmt.Errorf("settings.%s local %s, remote %s", k, lv, rv)) + } + } + + for k := range localIndexJson.Get("mappings.properties").Map() { + if _, _, ok := compare("mappings.properties."+k, "mappings.properties."+k); !ok { + errs = append(errs, fmt.Errorf("mappings.properties.%s", k)) + } + } + + if errs != nil { + return fmt.Errorf( + "index %s allready exists and is different from the requested version, %w: %w", + name, + ErrManualActionRequired, + errors.Join(errs...), + ) + } + + return nil // Index is already up to date, no action needed + } + + createResp, err := client.Indices.Create(ctx, opensearchgoAPI.IndicesCreateReq{ + Index: name, + Body: bytes.NewReader(localIndexB), + }) + switch { + case err != nil: + return fmt.Errorf("failed to create index %s: %w", name, err) + case !createResp.Acknowledged: + return fmt.Errorf("failed to create index %s: not acknowledged", name) + } + + return nil +} diff --git a/services/search/pkg/opensearch/os_index_template.go b/services/search/pkg/opensearch/os_index_template.go deleted file mode 100644 index f904f4c12..000000000 --- a/services/search/pkg/opensearch/os_index_template.go +++ /dev/null @@ -1,65 +0,0 @@ -package opensearch - -import ( - "bytes" - "context" - "embed" - "fmt" - "path" - - opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" -) - -var ( - IndexTemplateResourceV1 IndexTemplate = [2]string{"opencloud-default-resource", "resource_v1.json"} -) - -//go:embed internal/indices/*.json -var indexTemplates embed.FS - -type IndexTemplate [2]string - -func (t IndexTemplate) Name() string { - return t[0] -} -func (t IndexTemplate) String() string { - b, err := t.MarshalJSON() - if err != nil { - return "" - } - - return string(b) -} - -func (t IndexTemplate) MarshalJSON() ([]byte, error) { - file := t[1] - body, err := indexTemplates.ReadFile(path.Join("./internal/indices", file)) - switch { - case err != nil: - return nil, fmt.Errorf("failed to read index template file %s: %w", file, err) - case len(body) <= 0: - return nil, fmt.Errorf("index template file %s is empty", file) - } - - return body, nil -} - -func (t IndexTemplate) Apply(ctx context.Context, client *opensearchgoAPI.Client) error { - body, err := t.MarshalJSON() - if err != nil { - return fmt.Errorf("failed to inspect index template %s: %w", t[1], err) - } - - resp, err := client.IndexTemplate.Create(ctx, opensearchgoAPI.IndexTemplateCreateReq{ - IndexTemplate: t.Name(), - Body: bytes.NewBuffer(body), - }) - switch { - case err != nil: - return fmt.Errorf("failed to create index template %s: %w", t.Name(), err) - case !resp.Acknowledged: - return fmt.Errorf("failed to create index template %s: not acknowledged", t.Name()) - default: - return nil - } -} diff --git a/services/search/pkg/opensearch/os_index_template_test.go b/services/search/pkg/opensearch/os_index_template_test.go deleted file mode 100644 index d1bda9d37..000000000 --- a/services/search/pkg/opensearch/os_index_template_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package opensearch_test - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" -) - -func TestIndexTemplates(t *testing.T) { - tc := opensearchtest.NewDefaultTestClient(t) - t.Run("index templates plausibility", func(t *testing.T) { - tests := []opensearchtest.TableTest[opensearch.IndexTemplate, struct{}]{ - { - Name: "empty", - Got: opensearch.IndexTemplateResourceV1, - }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - body, err := test.Got.MarshalJSON() - require.NoError(t, err) - require.NotEmpty(t, body) - require.NotEmpty(t, test.Got.String()) - require.JSONEq(t, test.Got.String(), string(body)) - require.NotEmpty(t, test.Got.Name()) - require.NoError(t, test.Got.Apply(t.Context(), tc.Client())) - }) - } - }) -} diff --git a/services/search/pkg/opensearch/os_index_test.go b/services/search/pkg/opensearch/os_index_test.go new file mode 100644 index 000000000..c1a9b9f4f --- /dev/null +++ b/services/search/pkg/opensearch/os_index_test.go @@ -0,0 +1,62 @@ +package opensearch_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/sjson" + + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" +) + +func TestIndex(t *testing.T) { + t.Run("index plausibility", func(t *testing.T) { + tests := []opensearchtest.TableTest[opensearch.IndexManager, struct{}]{ + { + Name: "empty", + Got: opensearch.IndexManagerLatest, + }, + } + tc := opensearchtest.NewDefaultTestClient(t) + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + indexName := "opencloud-test-resource" + tc.Require.IndicesReset([]string{indexName}) + + body, err := test.Got.MarshalJSON() + require.NoError(t, err) + require.NotEmpty(t, body) + require.NotEmpty(t, test.Got.String()) + require.JSONEq(t, test.Got.String(), string(body)) + require.NoError(t, test.Got.Apply(t.Context(), indexName, tc.Client())) + }) + } + }) + + t.Run("does not create index if it already exists and is up to date", func(t *testing.T) { + indexManager := opensearch.IndexManagerLatest + indexName := "opencloud-test-resource" + + tc := opensearchtest.NewDefaultTestClient(t) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCreate(indexName, indexManager.String()) + + require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client())) + }) + + t.Run("fails to create index if it already exists but is not up to date", func(t *testing.T) { + indexManager := opensearch.IndexManagerLatest + indexName := "opencloud-test-resource" + + tc := opensearchtest.NewDefaultTestClient(t) + tc.Require.IndicesReset([]string{indexName}) + + body, err := sjson.Set(indexManager.String(), "settings.number_of_shards", "2") + require.NoError(t, err) + tc.Require.IndicesCreate(indexName, body) + + require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), opensearch.ErrManualActionRequired) + }) +} diff --git a/vendor/github.com/tidwall/sjson/LICENSE b/vendor/github.com/tidwall/sjson/LICENSE new file mode 100644 index 000000000..89593c7c8 --- /dev/null +++ b/vendor/github.com/tidwall/sjson/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/vendor/github.com/tidwall/sjson/README.md b/vendor/github.com/tidwall/sjson/README.md new file mode 100644 index 000000000..4598424ef --- /dev/null +++ b/vendor/github.com/tidwall/sjson/README.md @@ -0,0 +1,278 @@ +

+SJSON +
+GoDoc +

+ +

set a json value quickly

+ +SJSON is a Go package that provides a [very fast](#performance) and simple way to set a value in a json document. +For quickly retrieving json values check out [GJSON](https://github.com/tidwall/gjson). + +For a command line interface check out [JJ](https://github.com/tidwall/jj). + +Getting Started +=============== + +Installing +---------- + +To start using SJSON, install Go and run `go get`: + +```sh +$ go get -u github.com/tidwall/sjson +``` + +This will retrieve the library. + +Set a value +----------- +Set sets the value for the specified path. +A path is in dot syntax, such as "name.last" or "age". +This function expects that the json is well-formed and validated. +Invalid json will not panic, but it may return back unexpected results. +Invalid paths may return an error. + +```go +package main + +import "github.com/tidwall/sjson" + +const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}` + +func main() { + value, _ := sjson.Set(json, "name.last", "Anderson") + println(value) +} +``` + +This will print: + +```json +{"name":{"first":"Janet","last":"Anderson"},"age":47} +``` + +Path syntax +----------- + +A path is a series of keys separated by a dot. +The dot and colon characters can be escaped with ``\``. + +```json +{ + "name": {"first": "Tom", "last": "Anderson"}, + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"first": "James", "last": "Murphy"}, + {"first": "Roger", "last": "Craig"} + ] +} +``` +``` +"name.last" >> "Anderson" +"age" >> 37 +"children.1" >> "Alex" +"friends.1.last" >> "Craig" +``` + +The `-1` key can be used to append a value to an existing array: + +``` +"children.-1" >> appends a new value to the end of the children array +``` + +Normally number keys are used to modify arrays, but it's possible to force a numeric object key by using the colon character: + +```json +{ + "users":{ + "2313":{"name":"Sara"}, + "7839":{"name":"Andy"} + } +} +``` + +A colon path would look like: + +``` +"users.:2313.name" >> "Sara" +``` + +Supported types +--------------- + +Pretty much any type is supported: + +```go +sjson.Set(`{"key":true}`, "key", nil) +sjson.Set(`{"key":true}`, "key", false) +sjson.Set(`{"key":true}`, "key", 1) +sjson.Set(`{"key":true}`, "key", 10.5) +sjson.Set(`{"key":true}`, "key", "hello") +sjson.Set(`{"key":true}`, "key", []string{"hello", "world"}) +sjson.Set(`{"key":true}`, "key", map[string]interface{}{"hello":"world"}) +``` + +When a type is not recognized, SJSON will fallback to the `encoding/json` Marshaller. + + +Examples +-------- + +Set a value from empty document: +```go +value, _ := sjson.Set("", "name", "Tom") +println(value) + +// Output: +// {"name":"Tom"} +``` + +Set a nested value from empty document: +```go +value, _ := sjson.Set("", "name.last", "Anderson") +println(value) + +// Output: +// {"name":{"last":"Anderson"}} +``` + +Set a new value: +```go +value, _ := sjson.Set(`{"name":{"last":"Anderson"}}`, "name.first", "Sara") +println(value) + +// Output: +// {"name":{"first":"Sara","last":"Anderson"}} +``` + +Update an existing value: +```go +value, _ := sjson.Set(`{"name":{"last":"Anderson"}}`, "name.last", "Smith") +println(value) + +// Output: +// {"name":{"last":"Smith"}} +``` + +Set a new array value: +```go +value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.2", "Sara") +println(value) + +// Output: +// {"friends":["Andy","Carol","Sara"] +``` + +Append an array value by using the `-1` key in a path: +```go +value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.-1", "Sara") +println(value) + +// Output: +// {"friends":["Andy","Carol","Sara"] +``` + +Append an array value that is past the end: +```go +value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.4", "Sara") +println(value) + +// Output: +// {"friends":["Andy","Carol",null,null,"Sara"] +``` + +Delete a value: +```go +value, _ := sjson.Delete(`{"name":{"first":"Sara","last":"Anderson"}}`, "name.first") +println(value) + +// Output: +// {"name":{"last":"Anderson"}} +``` + +Delete an array value: +```go +value, _ := sjson.Delete(`{"friends":["Andy","Carol"]}`, "friends.1") +println(value) + +// Output: +// {"friends":["Andy"]} +``` + +Delete the last array value: +```go +value, _ := sjson.Delete(`{"friends":["Andy","Carol"]}`, "friends.-1") +println(value) + +// Output: +// {"friends":["Andy"]} +``` + +## Performance + +Benchmarks of SJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), +[ffjson](https://github.com/pquerna/ffjson), +[EasyJSON](https://github.com/mailru/easyjson), +and [Gabs](https://github.com/Jeffail/gabs) + +``` +Benchmark_SJSON-8 3000000 805 ns/op 1077 B/op 3 allocs/op +Benchmark_SJSON_ReplaceInPlace-8 3000000 449 ns/op 0 B/op 0 allocs/op +Benchmark_JSON_Map-8 300000 21236 ns/op 6392 B/op 150 allocs/op +Benchmark_JSON_Struct-8 300000 14691 ns/op 1789 B/op 24 allocs/op +Benchmark_Gabs-8 300000 21311 ns/op 6752 B/op 150 allocs/op +Benchmark_FFJSON-8 300000 17673 ns/op 3589 B/op 47 allocs/op +Benchmark_EasyJSON-8 1500000 3119 ns/op 1061 B/op 13 allocs/op +``` + +JSON document used: + +```json +{ + "widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36, + "style": "bold", + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } + } +} +``` + +Each operation was rotated though one of the following search paths: + +``` +widget.window.name +widget.image.hOffset +widget.text.onMouseUp +``` + +*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7 and can be be found [here](https://github.com/tidwall/sjson-benchmarks)*. + +## Contact +Josh Baker [@tidwall](http://twitter.com/tidwall) + +## License + +SJSON source code is available under the MIT [License](/LICENSE). diff --git a/vendor/github.com/tidwall/sjson/logo.png b/vendor/github.com/tidwall/sjson/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..b5aa257b6b5a72bf7d759c23d94c5928cd8eb318 GIT binary patch literal 16874 zcmdtJRa{k3_cjWGbcuin(k0#98$lZBknRTQ?iQrGTe_tiX#@r7?rxOk%-gV8WB0>WI)ax8N8VkYJmk{%8w z${zA6Mjlp1yvF3hLZkw2d|(0_6DI>wHydkPM?N<}@_+O4fzQydnaN52UE*XVNdBKi zX~=yb6|-|NA?0M^Vl-l9WhLe2Wn$&zVQ1rHAZ24=4nS65diS9T^l2Qy|?US3{i7B*%!Hb!s-qocd6 zlYtwfts}*MXLx7gXyjmF?_^Sl<8Jv3z028_ns zT)dnHtR@_6#zy4-Gv4_BV!i)(Dfk9FiT!^U#AnLE!fR;6Zpvt2z+=qF%F4~l$ZN>X z#>mUf#m-}7%57-GZ9vWpmdFfsod1g>{r4;2ozRc}>sWwa{@1}Wu?45e0UR6n(b7y9 z7{N!WccLn8^M^}zvZ}5fC!ahVA>~F)_WjxrcG6uq#v#%Pq1-p9kz$4t6$?`tMh*fm zBBO>XqE1ER<~MtUq|(DM{mc~%=aIJUv42Pr)>s-rUguzJN#Pn}qhfB_<*)VYuJ+aR z)3obp5QU(uFQ+%m+&=nr{-qC^_<=M1SkdtJ)@S-;MMn7HQW+XrUd99^UUdJ~>11cLg5O+(bWXz4qm3kM z^oJ1s8|{auHYP#Avg(Pun=LBgUPJbcjdR=$@!I2L_3!k-!RL17VlaJhu0~y#r!(gl zmxE*9Pppz#gcvDWyJS!=gmcA@LR9{FME{%h`qfEjZtVTotoxB(Psvp}g`u)^eX8ur-T7*$hIJmc4~{}Edht!md2V=M7&htJj;>z1 zm(6UG#lp$c-|b(%x$LmRf~lE_d2?yU0ka3fQb=(x-EIfBYRFI4oYXn!$a3q8La3sh zI+E=gzhSSfetr^t2`^E)`DOAZ6TS7#5Q2=L2hW+!-QJhUsy})H)WVLeB4%4NVEu{@P7U}%_iIZZZZr{?3P?SBhOwG^D?{@%f zV$Dpdh_Z+y%S1DbNz7HYijUHhR3OhX&Uk$|d~rt+^?V=2YbAo9n4D8Bs^vg8eZRff z^qG7rm0tSo!XF_z?D1mt2uKJ7=~M)nc6g0q0j};7`L!OBO}ZaPJl|vTVTlNQkURHF z=v3)4rZH!-{eg@70!+RN%vV^6uoHak?ni|!p8mu1+v<|vd8%bI$=m0!d7sF~vO=l`4}%bK|6 z#IdPE@~@cL-Cu=s5Y@4qe}N~)nx+Zuh)hCr%_^py9!!s6aoQ3~ zd-izFht9CD#Fdd$Z;Ky}qhxdUcjOgrqA+(~rwhdd4w>#RTlv?|&O}~FJy-hq%VISj zJU|?EAt9{K5^bSI?qt{B+wm{9@Kp*!nW?z`XDu)Ai*U8>CjDX7MK!=`4+MQr7ye9g z1V3F3Y<}38(;1cMn*ZF1U@A);O35^NAS^u#llSVRm*o zp&|uk|Hqbu_vS;DPqJQ@$72vv7+IL3dkgXB?0h{^olc~FZdlS>`j;{=$5#zf%Iq$v zXL0Ka9p<4h`<>q&?#4>%%=!m}b4|C#cDp>|G2pQ{)qt_&q5s=2?kLSp6c`9 zfLK(pl{n)o@uuKAvVT7&71;z5DRB(CHFcB%qzknU7Bi%moUy9;p-{l&N(o&ZshD(i zBn!HTK?F+p7<3VcMy{mcH9vB<9P~j%xJb$Tto=^MgG&*UjNz~YMxeguXDtHp`wZNI zhVZL2l1Io{47w9_kX1PyAt}_4Pv=HAvF1c_KOPlpj35!yKF)eb89e-z5`R7ilR-dL zF$RZ&g?T@UkZI=n69YxiTY&rW`}=yhmi+_GVo4halIsA+%>ho%u$3C#HYQp+On}?! zN7!3A)Q^AY5?h3h8UoGC5eK9m-WNA(eOFbziIogn3$oUKH)gqBxj;>Dw3nNV+XqK- zsVC5XirX-kbG{jcPRRWehbjMYlA)~nR76Hb!*?m2cx+RhVz4#k6La@91*WoyfJXk> zj|*iubqmByUH;jaSY;9ZrqKn0)LR7`F1O;=bZeO9ky5Bjbf=BVVt6n3q`6F#tXBMO zOd!r9dNYF?b>62~?7X*!sHL^(sex!;@h!=!mw~|;2Onw_$%)AU$958u0wSk4{SVbA zaP!@tz-FcV{5qXGl3a&^+IxH8V8Eif^v+>AB?t@VKVv3dk$@;B%KtVYR{4X&=Ch+! z9=o^Cj&dgblaj`^whSJw5Ca1l#toT5@Sg|%MG8awluq2tyXc7K1?C>JA_8IKjYq}i@WDzm#fj|;Oe+>B15tl2 z?TsdBg|U%8mEP9vlh;T-V(VMxWIiyk?NAXId6p7a9vlZ?!TM;el zq#}N^En*8I9I0p;KEk@F>};^QBiA6)ybQm{v01rf{zS{=rC2g^Xp_o|NkwaNvv}3W zXCjCov9P{A78pkuLb0J0>|-m+F`qchJU)(GT*SikqpkQOh@|hFX`BxS+ZXr>WT1db zxL~%%WBn3xHxL}(!s7Bx{K8%b3qYf<#py+Y`!^%=sadVR5I_ee15fcTBEygwQ9VcY zqnxoHEiG8dbkYVIvF#c$yvrt5seII1#<2 z?AAB)_HG)Oz>S*sFfiWSW+JJ*+BhudpS0WD3?UREve=v{Ern$8z_0u3j`Dr@dcKLC zMAwrCGWJiTQe#}aV+(7vzxdW3f0~=yw9`1O?*3dZE~=xT>Sc`dD5_kSR6G|+4#I?e@F!hci(5+fY;CkPPUsrN_n&|V2`t{e$+l$G8ItAa1$xb)= zk(BjUb2$yazw8fq5;G)btN7C|7Vm7gA@=@7L`)QZk(_)g@9Mg%@J5*1OCy#;--i+k zMvMd92q)7|YMMkFv{`@;N_cv1waP8gdXJ42a&ihR6vc`PAkDXTa5=VO0BA^QuI_4qb|sR8O;%S+Qr9IOp;#-V zq#I1$%&S?}RV{0JJB09ydYj3XLR}o-7NIX0v6E*PNxvBxv5U7DjRuB=|L4AY#FI0o zi>$?!3Jcn|%paGpcY;W2WRgzvz8h+#5I5gEbTv%u=o`N}@_tx-%_PGRcRSk$p((4b zzU&LXm!w={^YHmP6o>0y@S@+t1Hdxu+lh;bWV&8HKJC5SVF3qesIf z7*m!;>TC#J_XXy-o78A_>o1(FEUCFKuJE?%YMo}y$w2Ur?xH_91QZY@v3_UoV72Ma zqx|iAj=E)K=G~s)4UM*%kk|GNtu=~l3cFF(>j>@uE88{JQ*z=9Td+G<-T}kV$1K zW~jEVzO>|Zx;g(@{kOvL$SM6Ra{&zoDY3cJN%|AZI!~eOiq1^-q5i3CO-IGK+veyJ zIL6PAkd5KW2MN9G`f{cB7_K~GuJqIl*8G$3X!!~+(ezdP+Bj|;3DV}vOZFWs3-~bq zHkW7cH}fREYlW}ww1WW%6uVbMAm#7wTF@h>ml9sc;(czVaCxHFw*+DoQpz@$YX8EB zY6zM6V1aQ-`WAgcY%8#nl1G*n60%ejc_@>f+OK&8T-IN<#XY+m8mxtfw$=&p=DXJp zNnF%!fo@n>6cTt+&K<3l!j}X4x{_`WtEt9sq`E(z)Tz8sGwvHCM@GoSj7&;Xfb0cC>B;K(DN@d>2)|!fB>S#TI3@*`74be7`618f8t)m@AFYx z46P^8ZfbE2x!n%R26li6e@BTp;RW6ei|CCVMjEVKIek-N*cBHM7~Z}erk_^?);1X5Dl@^4Oo*|iqy zWNE4h$<`Q0x1TOt7r+rM`>^bp!$rt2udUmWWI%Zn8&*2AYjd5`dB|mQTO}HL+c=zv zJnVnn){-VcO3WjD(-*kXan?a&N=9A>=tf4+)YYYB>*K1c?!9)X^FU6S z{e!ezTGApd_A=>PLcl`K`*R5=CkD>>$=<*t{rS1R_I*)-)9%rPWhyz4E>RqWYdMD( zTk=6_Zo<&0_3yk$H^{zgQKzL9@|w-*zns>IqE~tARax$9YH1-C^&0|b4?ETQQ;9O;|L%?3*N zo9y8)Pc$cdeH!pz1-}VQwqW4dFfr?StCxd<5hdC3cc*a_t)pVr-C9Pv)dUtr|EuSL z|0Xe|zLFOh9DJ;3MsJmd#60aq?V~cN25GDmM37(+=pa?H5vp1)W?o&PMBgMGOATFo zR3qf{@)ZXWyXY{e>pHJ%!e2zdFU>8cZfSSJR~Hgyvil-Om^h_r>G0gG&Wkvptx`}c z^anjNqKrZYixr&z4y(yA{Kn^J?DLaidgy0eWNACZ1s&G+7gt-J1l5tW&b9ejT>Q?EUMbqp(A& zp#P+VPwqLMs>=M5kkg{qZ&!$AZhA+4y`-7ejKvo$aw1)9cL;8@*S}c|V|))*&dR&f z5Qq+f;LByr*FSC*|6(Ui^Gy6&N9Xt@U?_bzNdB<}!{>4F$CxS)w>NLmC_>5S*GPSE zLGl+NXlL_E9$ydL_;+wV9DpdfeMUL_9#4wOMtR16Ak1P~!PHC&NwfofPlS}X=~D&$ zGTRMcSR8boi9{e%d_J_gh+NdBnBK1SycGrBR{DjO4=W5&-tE>+{-|bd$`fWb)GWzA z$*;ZLGMW*%R)9&l+GPlG&HIan8kd7#zf>}xLkA$uJ*l>pV*BBk$|FiTp7l)*vl1T!YuO4wnzY|ulte^SJuAEqZ*YhTy6-TME{&|NT@o>`QI zy0htUBQfFQz7dn0NCkj}PNeW6jubm53udkxW}Dml3P^Yg!l!tyN!B_03{YngNdW~4 ztLW>A!|3vjSmWVjjs{QTSlSo69qn~~c+5UF%G$fYVx+Q-^9qwl_DL@$$@fGg>BcrB zxRR3>1ZxKqkSLu?`y^Y(A!-J>rQ>2yD{w7Te~Le+$RKE<)6>o&CH@Kueolp-aRcdT zC6i^}q8ZL5;eU0)27Dh0W2DaJhL#hg_?@@TUmgPwJKbb29?|?ZHn+V2cy^JDQv{Lq zOUCmD(ZOK1Q!UlRQn+~QuJN)-&qTUetn<-$1&(^^!ZOFkTKDEe#UEwuB0sHcRrPo*uVVM%5`G;-TGZbA zgy>600~d#gT68P{9folcf$Hewnc;|80#}~~r`zMrsC{Z46l-dc zhiRd+mtX#!du~c}I=VTL{1f};hbro;l5t^RS>@C!TUs^wrWA3{%|~nB{TJS8`&}_n>A5%nBu){Y{T;h)_wQwSDE+uVo!s z>?!zTM&h;K>|9xicYS_;W756cs1>STFQp4!aDIw~p3-#VO`L*C1@_C;;`g>bQryq1 z%`UN70nHQy-|_-`NIl}O4418Rr0~hRtIx0IZX`7JCO0^}*|7j;0d+x*?~52g3Bhuc zZMN{H6kTdX(HFXh68c;aZ4OK(2m51xH}QuUI@jSzZuyK4ZYuw5b)KFlSkxKx9~K6b z$~9kK92yP$sn_{wE(YLfRzGf;6R}1}ybj_rzv7LRD~D!weI5QY&88<1dh5Y zt*q6*WgB*1=b1quPv4D*iZ76ns{! ztx+8ASr;}}6tjn}q`v&KNYlw{a!3!Vg%8?&ZEri`aI$sn713o|W8e>>1Z6xm!&R|J zH~ny%PT!ezbsDnn^mq|Tzuf7<;sY<0Z97RGb(Oxr#)KWTZ)!bT%Wy%4iMXjwUzqbi zj@FNk%m@HU_0T73xo_*q<0SF%^T4d_>;5@Sb+ZR6FBYC_V9&W=st-ELG@>>q{v$6j z1%~;tI20@uLO|@?3BrQLls~g-H9q~PW~VWI9V)86NoqWTgP6w=mK zdFwGVoQbnkHq*K8HRFHox4C*JzJq50wdrB3PQ5FrhL78HhY;ehfbXZYqAB11@W%)$ zRCQ42@Ze%r$O}-Gsa8(TfqDW4Grk-?&nM02GcNV z_bO9!RaA3+P=}-fcbrx77(9~z<CP7Ah*1lPtLyvGd%oc~H~o zq7gB*GiP+QRSPfH3!uxy974eOq3At`AziK%8TQC1_jv)sw$im~@<^e>tXcku=2YI8<^XVxKfGrTOn{C^>8f zC@+xqnOm+}0=n^=N?-_qNpx8meuWRsafAdRUJ(^OpR4A@HP$vU7A8mbs|MuUs^-)} zoG`>BNu6x9Keqfw)#P?EoOZ`-75>$O?Gsz7k!qO8fDRu|dxPx}!bPzlW(!;OU+65`|81VlPCmq#$CJ+iIFoNKB6JpcvS4+obyU0K9o&C)YU0GfGuU*a6D!e1W*6(VzsTp~kE}8=#C8Hv$6whOq`)XT2ivU16{oZ zC{s*N@(YKQY`45nsH{>*U1jq+BBr|E`}>_Kys4Q^=Dg8GA)NosSs!2u&j8a(c)Okt z_;^`K7-Z3607*bbl$1w$7&^eG6&Fmx=E_l1Y!lZ)VFzdY8+FiN>Pbq;n<|i=_y}FD zvV`AS_)mj@xxnFoYlB7w$8)uAIs;QLG`g&uQ8eJ&nQiV!ltJ*|G#v3f?A!0Sr#%~z zM>-6R@dE`40HwkeUst!$@#QY?EdC|XL-5CxBtO8M1Fn2&uKF|Wh-B%#WY7Z^HbB9S zT+CGPGTyk`(>o;>gm32-?0pPdD_0Tw^{Tk0p zLd2tB%%~Q_)xmN51}7?{q=5uZTr(gzk17vV)Z8*oAb>qY;0iqK31CgVcN6J!d%~c0q=)j0M0Ul5WURnPiHh}1=$^${I=bI@!ofLFP!`b>B*uvF z!xU9lfy#ux4^H`KR7^G4D8BLIG~#9{Wm5ND360Od6@V|hZ6l_fe@Z3^|0>u9&xdur zNKK4CdLZ#tz}X0d`Fe>(mP004|JKJ(X<9iIb9v5TXp=T0ADiN5{4j5oJ)U z`+o$HF)X9`=K$S%^&`(J>K|l$JacdjAv4zfRae}ihML^YD zi|ITi{SxDNjtt9plKH;16;r1})4I|!Oz&Ed8F1jq*8)ra>&n*TL(n4yUY>N8|yl|d|=_K7YcEH2YC*?=B_ z-l&hVOla}ng>}KsQO8MdnnfhT1j(xBX`j{ckcnROa@kNBEj&J0Jqe;6|B8NN&a_2@ z)X1!}{^kW{Ol{#MR$Y?OW`)b}7Yn3oHIJOa;211Ghh#OJgvQ7Vbj=3{Mpg5T5h0zw zCf5Uof!)SEyd!nk=7&5(h7E$BT(Y&1(+{QSVy?o$`-Q}OFEV+r_-7!+jI?_C%~_WZ zHd1{VV8n@PWfJ_0tu7^WHw$!PrXt=&1iRLnZ1&fijB_m#)i>a@(C%1OA{X8@C~@) z|B*uaU{&6zepGYj6B!F^4IBbWPJ=~f5n{W{T-7P`IDkXvJ?RkXBC<zz91#}H2qva6gyaObAP*DET zQhD)OZ0p_Cyw*JI2QBrt$09Kb(XU*DYmwN_V1%I+Qe)Q($7E%^5d14W z-RZ5PEW5xr&rK%j(^6MsgZ8)5&yjY{J_y3CiGM6Gu{M1EY8@7u$+pLInE02Euj`@k z_WU~)tW8_&i@+KgSyC|mrlaeqfReaUNjXwP6C zH5M7k0Hmd0PQ`%7r=;L^1>vQnhV|z7NBoZ_1mw4REz4&Oa-8EG)owql3;uLk9E&t9 z{5)&BicJlX7Z6zlNBb?X(2x3{wJM?Qf?m>#zZ+=;c3J_bsJYUgjK&yxNtSfKWFA>p zyn{9%Fg}@S^?<b&Ojpgb-Xa;4Nr;IExcCW|i53mITkph!mVcLKiE50}5_AA5b78xydj+T49 zj!^=HKDg__1*Nf$7}7Zr>}NZ#g5Vnrz2LukW&Do5BsLdsZ<07q&98$4nbGS8rTB*} zwU~g&FR=iV-YC%i?@xVBCd$Z~pcRY=?T*3vd;?BXyKw|uEk%gT|4h7E)rZo-!CLkxlV1W*F-Ekk zdx8@dE{2e!nwruQJ>aB^THjnzdWQgq0s(!Fg%}Dke1!Te4ah}gOBGyM7MT3@iSN5o`023vBR4UfNBXZz{x5C_zRHs z$^f5!`b;1feIGuF_(~GmHRKn@W<97jDyg^{$f_|ZtNy6qsM(KyY23${8E&)$PIKlbDH7LKkMTk8#`#L|J_ga8@8)t!qFqE~SpT=$nH+;BuvGWe*> zpQ||lTn1u87^0UGF$rUWHs&ct?=Fd(-tS-|eSVoRQ{(XB3}#JQC;N{?1M3Br#xf2i z{Fota`Bjzjo(f{XIZt2P=H?dDjlSab^u;#w3)FB#r0khc{k~iw@6oPzJZV@s%Ofx8 z%db|hPjs8te-sw-qQ;`fN$*TiyL_M~7Ey-)=Kl4=05AK{B^=Mbh|Fgqg(;0T$(hf; z1}=C{?N)msnt+`t0-&0`@i>PFW(>L84~?X^19mtYWkkl_4B2I@9kJXt z)f$B#s||VB3BtMm6t`8jOd_Pi z99aRsT7`&=0LH>l6?I~3ZQhR@z+p3tIcd zA#O0@-yR-y2w*?R*lzY86}1^|72hKfz1Az>eg4xKD-yIT0l8UElo+@W4Q}dj>vjx0 zaYmTd0R?UTEqZ0R$!f$Eu+GtEL|TkU`A2nn@X~6- z_|@?msP4d78OO!-R?|6h<9|!y^kPHI^Ea#)70^@X&B9`9Y#+LU38uuCgfuP9OG75KKMwHxR(+~l1K6bXWx%%jdoCn=B0$WP ziztbn-1utU&z%Lu#Ez|!f2AnVuStu|27Th0wsq-R6Pp#=TbV(21LNlJ8=vtSi&Z_` zS!&;EKnt?!08q|T;xfpg#$gq`w{4jtC^?vHqRfeyVg+Pp@*`n5O16p$PdXMG4l^3T zM%yzA>a1ZC(*vhh5_oH`p);JKf`orHWq=)(t%aJx8HGs_zX@kjw?; zxUcxl(xXveL)1AeSFSeDe@$kvuuq*t8j!f}fYxdOQ5Yh$@AMA9xA8l@Enjjnn6-T? zOOIu8*ElSoqG_({t}uzj7a=#P_kN+Wtel1!O(oZ>q^&B>LP5N`15@p!@iXEnRP`M0 z83dZoC&*KnEs{S#Pl z28gwyO#-^xJHCNBR$bu*Qh$|CrWgeeRg!o_+j9j@%%sz$ly$*C-7oI;v9t^8F5>xI zVi)ymlODofs>DPhwn5Rr#9(&0i`Y&(7%cd@-^0`0 zvxotWKBAHOz{>J=Dr#tW8m(yFdhj+8)V=R<|IYSZ5i33-&;ohZB=e;*XrBY1E2b_a zyZXAhF(dZA(S`fR5!x zpL6Vyd0~%|K=_(~Wg%Y|Io`o3H z^vD3dQW*uZ)0R~1Ni_ZT(Yjb`uC|WdSn#B>xDQO2a&e+~o2^ARM_S`vh!6$%JdN*z zG8EqssDFiiL_0aNifMb9rV67K0U+lfg9- zWFbL-5ghAXkLMF*EH_bYo5L6u3(zi;m~yuwAu`?-lWq>GM_-LSxY1UOrKQ*fs9Il8 zdFvl3B(Z$QP^Kf)UDJeOj7Cdt1^_r2@u;`u3PhH=WO!{RWD#9RK>;ie`ZwEPL z9YTv1>z~O)*D~OFDV4cgKi=(qeMP)bCoD32FknOgFmnwR3UT0K7YGtaz3#?t&Y_)!_B>An2`6192{$lXjN z8f^tfBuD&4?B|JuiMPMSfQDe6{`*WMqi9>V0QL?<7@>~(_k^>41Zak4&;o9<;F;QW zR|nA5+4U)El!ZwP-LPq4+ZIb}1Kr7{(TQke`VVRUwSR!a(k@k(`5xkLkXSnyHjjzP)kU2o7?~9Zo|Nf^}Q%_CR(#Fdd^}I$$Fy z#I(%w3Br_a%rH!;E~DpI*CJ2@SyQ~nM_Lh%gy^BlZ&@>pl9&jDK| zxahvTnl3Hhy<+z2et(8H4o^u>KHfL-n(+mBkc~5j?f7>YOkdgzC_(zqZZ~^434nZm!Hw4tK&feG9E4$ zwQqPU>E7#JKLmeX>(1l{VsQz(O$H2|j>D58Q;9bj6No94uw1zgYY)YNAY)|kk(ZL;MAz+h!xyOPV@ZI7sX!Ah zRQK!o6go+(GLVtQwKpa#KI%Q16q;+}Yseq3J23w}dsSJ3`Q9e%oETe1e{lBF+BcaJ z_9V-)`zmH?H^opLu_vONt*Tshp3!?PK(pOQG_A+8mW7{{Y^9>vZPW1_)W{zmInMN> zv=2n8If?Hxt>#Dv(c|EJbUr`86uBbx{UEg8+F)+;-czcLo|`x$V{S^eTRGmai`UoJ z>(>krsnMuQ*W!xC6FJsK(VV&-Ca+<52~Z6%pFwzZb^Pn8BqJuN+RU!*$D z@oZJ?j&_7NyR}?@1@TeMPHp-a_VkkihnO8g99-wDwtS6hOlbs79)OyuvqfBK#VGeOB$7qm-H_E*=DLAIihG?`@+Gy^H#HRy z$zr+v(*ktyFUu<0FdFZ^P&W`nYBb&W-eq%8t^DQ6#@i+ z9pH&WN7HF~IdZe?yjnrBZZ(72Vb3BhZ^nd@&H` ziq&7eD~F?%CDQ)%E8A^#V36FxHw1UC07%Qah66xyC03fZADfzDDIXNFi(d{$v_#jr znaK2p(O$i)&cl(wqmq{HYFPQHIW)y~apmFd9pkI`_(m*Jw$pTWw@bPkjSc>c)E3a6 zK&c4TF5U4{%sGw}KA-Yd@wz^p_mu+L-fD8IAWp9gG6@Mic2Hs$5m{gH^ZgA&MDd>N zkuLf(I7oZyt)V#FQx0d^M{Rn9Iu6=Lmz0B)K=dT@Rx&__*YEYjWzs2f9sdO~+5OaXc{Jg|BN zHRT86rVFDVm7FNc9>T5Wc5$_^6rZtMtEy@+;`K#X>O#1;NH0}s+Z7KPzF0^P4Mx6j zW%(2Bmh&}7Rwuh3`CAttSBFv@P8|jGjTIr&%g&^G0qb1^N(uCu{E`?(n89Hl4rwW0 z4(DGpSC-}h^uIPTNc&a5JupbW1%%VMnz)jkp~FKPu+q1L*K9#z$9RPf!r7b;3>gt7P*yr9gb;Zf`LFR%I@Bo-OU)Dk+7&u5{ep+#W09~ zLV|K!UbUl0H3u_)Ow>jnj;Lil+f)K=%>?Amk0rv9rsK4+8sIc>bLRCb(I)V z5KiFOLVKV&VN=OUCbmG?Iz&y#aAoY01l|tVdQU#25&XfY7ze$aSB&fQ0clPy)fz_J z)dfmO&SOJ{Vo?F1(0c=hkP}j#4rR6ZizxB6akGHp)ELfJN>Ca?N+=Qy@~AeThjhNF z5<_Ku1q~N#$-oE!smjYh6k;_%H7Dk`nM`tYT=X~9di6w{N|m-I-WrB#+pH=R$cnKr z6LgYM;}hfS1B9k+W%hE1l|_vMK{pW~GY+>y7j7610|W-<}Z=bj}B) zY|D!t-1{(Q3B2!#=*xS@7zb55@5zh%``va4TY$-2BCq4D!gH;2qx85F%2hBWQTzm&F{kV(cIR(B|*HWfn z9Oe7&w&o1^Ktud>o6)l+y+)$@+`|r{X_FKyUSu3-Ce63vYTUcuwoG`J^bk9&%0$!O zPDv{}e3wup!k|7v%!Iq{56caRs}eOJ{;WH*yEy$lS7kAWfni-jp$%R=TKv&udM7l#5N>H8j6 zRK{*d2C;iGg+SPbP3-K$_??&6H}SFzskN7DR(P?Yam6oYwH|OYpb>=Xu=fiW2*z1z z%@flW%3GfbI zAP-Uqe^pch^t^Lvlqx z^xjGWVQb@j-yUcwIpM|%tK(z3I`H5FIlZT%qg}=GgWzWZV9!VyKphDhl~|b27oNZi zc=abMp`Evlpe<=0wFSCZHg^xL=`xrWgb7rQ_m1M?eSw7qeDw}z7U784*wFTWOSY>p zy^vRmzKA3#J~(1AZlq-XZlE$)->#WR1>GIhXK)piKOU&SKR>x&H2(q@@it_v49p+WvRnH z?P>(CL3!~&<|91s%rYr?5(cGZzt5N?Wz0zb91}HozXt6HRMw4$@902!fJFAEqUdmI+xhvu z_uZybJe*jwk8B1Cv-(IG>I_4a9=-|yVuCpn^yc6mWM|)QzdwcT1-0mCq<|?D#uHau__9&o6`=rF3_}&<>Eq28;JBPjt(0kyc zaa>5pMguuL#RJ4*~E z6lSbqU4E*a@~quYjQKnAD{4~Ot*P_S(*1N~v)NYjgEm?`^!;F26wL9~Ip(X;sO(lD w7#LDw=!>=gZ~Vll8TR$~+o={m}9KMaKQ0E4{g05dmT>f1pL_gsF0EzfATL1t6 literal 0 HcmV?d00001 diff --git a/vendor/github.com/tidwall/sjson/sjson.go b/vendor/github.com/tidwall/sjson/sjson.go new file mode 100644 index 000000000..a55eef3fd --- /dev/null +++ b/vendor/github.com/tidwall/sjson/sjson.go @@ -0,0 +1,737 @@ +// Package sjson provides setting json values. +package sjson + +import ( + jsongo "encoding/json" + "sort" + "strconv" + "unsafe" + + "github.com/tidwall/gjson" +) + +type errorType struct { + msg string +} + +func (err *errorType) Error() string { + return err.msg +} + +// Options represents additional options for the Set and Delete functions. +type Options struct { + // Optimistic is a hint that the value likely exists which + // allows for the sjson to perform a fast-track search and replace. + Optimistic bool + // ReplaceInPlace is a hint to replace the input json rather than + // allocate a new json byte slice. When this field is specified + // the input json will not longer be valid and it should not be used + // In the case when the destination slice doesn't have enough free + // bytes to replace the data in place, a new bytes slice will be + // created under the hood. + // The Optimistic flag must be set to true and the input must be a + // byte slice in order to use this field. + ReplaceInPlace bool +} + +type pathResult struct { + part string // current key part + gpart string // gjson get part + path string // remaining path + force bool // force a string key + more bool // there is more path to parse +} + +func isSimpleChar(ch byte) bool { + switch ch { + case '|', '#', '@', '*', '?': + return false + default: + return true + } +} + +func parsePath(path string) (res pathResult, simple bool) { + var r pathResult + if len(path) > 0 && path[0] == ':' { + r.force = true + path = path[1:] + } + for i := 0; i < len(path); i++ { + if path[i] == '.' { + r.part = path[:i] + r.gpart = path[:i] + r.path = path[i+1:] + r.more = true + return r, true + } + if !isSimpleChar(path[i]) { + return r, false + } + if path[i] == '\\' { + // go into escape mode. this is a slower path that + // strips off the escape character from the part. + epart := []byte(path[:i]) + gpart := []byte(path[:i+1]) + i++ + if i < len(path) { + epart = append(epart, path[i]) + gpart = append(gpart, path[i]) + i++ + for ; i < len(path); i++ { + if path[i] == '\\' { + gpart = append(gpart, '\\') + i++ + if i < len(path) { + epart = append(epart, path[i]) + gpart = append(gpart, path[i]) + } + continue + } else if path[i] == '.' { + r.part = string(epart) + r.gpart = string(gpart) + r.path = path[i+1:] + r.more = true + return r, true + } else if !isSimpleChar(path[i]) { + return r, false + } + epart = append(epart, path[i]) + gpart = append(gpart, path[i]) + } + } + // append the last part + r.part = string(epart) + r.gpart = string(gpart) + return r, true + } + } + r.part = path + r.gpart = path + return r, true +} + +func mustMarshalString(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < ' ' || s[i] > 0x7f || s[i] == '"' || s[i] == '\\' { + return true + } + } + return false +} + +// appendStringify makes a json string and appends to buf. +func appendStringify(buf []byte, s string) []byte { + if mustMarshalString(s) { + b, _ := jsongo.Marshal(s) + return append(buf, b...) + } + buf = append(buf, '"') + buf = append(buf, s...) + buf = append(buf, '"') + return buf +} + +// appendBuild builds a json block from a json path. +func appendBuild(buf []byte, array bool, paths []pathResult, raw string, + stringify bool) []byte { + if !array { + buf = appendStringify(buf, paths[0].part) + buf = append(buf, ':') + } + if len(paths) > 1 { + n, numeric := atoui(paths[1]) + if numeric || (!paths[1].force && paths[1].part == "-1") { + buf = append(buf, '[') + buf = appendRepeat(buf, "null,", n) + buf = appendBuild(buf, true, paths[1:], raw, stringify) + buf = append(buf, ']') + } else { + buf = append(buf, '{') + buf = appendBuild(buf, false, paths[1:], raw, stringify) + buf = append(buf, '}') + } + } else { + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + } + return buf +} + +// atoui does a rip conversion of string -> unigned int. +func atoui(r pathResult) (n int, ok bool) { + if r.force { + return 0, false + } + for i := 0; i < len(r.part); i++ { + if r.part[i] < '0' || r.part[i] > '9' { + return 0, false + } + n = n*10 + int(r.part[i]-'0') + } + return n, true +} + +// appendRepeat repeats string "n" times and appends to buf. +func appendRepeat(buf []byte, s string, n int) []byte { + for i := 0; i < n; i++ { + buf = append(buf, s...) + } + return buf +} + +// trim does a rip trim +func trim(s string) string { + for len(s) > 0 { + if s[0] <= ' ' { + s = s[1:] + continue + } + break + } + for len(s) > 0 { + if s[len(s)-1] <= ' ' { + s = s[:len(s)-1] + continue + } + break + } + return s +} + +// deleteTailItem deletes the previous key or comma. +func deleteTailItem(buf []byte) ([]byte, bool) { +loop: + for i := len(buf) - 1; i >= 0; i-- { + // look for either a ',',':','[' + switch buf[i] { + case '[': + return buf, true + case ',': + return buf[:i], false + case ':': + // delete tail string + i-- + for ; i >= 0; i-- { + if buf[i] == '"' { + i-- + for ; i >= 0; i-- { + if buf[i] == '"' { + i-- + if i >= 0 && buf[i] == '\\' { + i-- + continue + } + for ; i >= 0; i-- { + // look for either a ',','{' + switch buf[i] { + case '{': + return buf[:i+1], true + case ',': + return buf[:i], false + } + } + } + } + break + } + } + break loop + } + } + return buf, false +} + +var errNoChange = &errorType{"no change"} + +func appendRawPaths(buf []byte, jstr string, paths []pathResult, raw string, + stringify, del bool) ([]byte, error) { + var err error + var res gjson.Result + var found bool + if del { + if paths[0].part == "-1" && !paths[0].force { + res = gjson.Get(jstr, "#") + if res.Int() > 0 { + res = gjson.Get(jstr, strconv.FormatInt(int64(res.Int()-1), 10)) + found = true + } + } + } + if !found { + res = gjson.Get(jstr, paths[0].gpart) + } + if res.Index > 0 { + if len(paths) > 1 { + buf = append(buf, jstr[:res.Index]...) + buf, err = appendRawPaths(buf, res.Raw, paths[1:], raw, + stringify, del) + if err != nil { + return nil, err + } + buf = append(buf, jstr[res.Index+len(res.Raw):]...) + return buf, nil + } + buf = append(buf, jstr[:res.Index]...) + var exidx int // additional forward stripping + if del { + var delNextComma bool + buf, delNextComma = deleteTailItem(buf) + if delNextComma { + i, j := res.Index+len(res.Raw), 0 + for ; i < len(jstr); i, j = i+1, j+1 { + if jstr[i] <= ' ' { + continue + } + if jstr[i] == ',' { + exidx = j + 1 + } + break + } + } + } else { + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + } + buf = append(buf, jstr[res.Index+len(res.Raw)+exidx:]...) + return buf, nil + } + if del { + return nil, errNoChange + } + n, numeric := atoui(paths[0]) + isempty := true + for i := 0; i < len(jstr); i++ { + if jstr[i] > ' ' { + isempty = false + break + } + } + if isempty { + if numeric { + jstr = "[]" + } else { + jstr = "{}" + } + } + jsres := gjson.Parse(jstr) + if jsres.Type != gjson.JSON { + if numeric { + jstr = "[]" + } else { + jstr = "{}" + } + jsres = gjson.Parse(jstr) + } + var comma bool + for i := 1; i < len(jsres.Raw); i++ { + if jsres.Raw[i] <= ' ' { + continue + } + if jsres.Raw[i] == '}' || jsres.Raw[i] == ']' { + break + } + comma = true + break + } + switch jsres.Raw[0] { + default: + return nil, &errorType{"json must be an object or array"} + case '{': + end := len(jsres.Raw) - 1 + for ; end > 0; end-- { + if jsres.Raw[end] == '}' { + break + } + } + buf = append(buf, jsres.Raw[:end]...) + if comma { + buf = append(buf, ',') + } + buf = appendBuild(buf, false, paths, raw, stringify) + buf = append(buf, '}') + return buf, nil + case '[': + var appendit bool + if !numeric { + if paths[0].part == "-1" && !paths[0].force { + appendit = true + } else { + return nil, &errorType{ + "cannot set array element for non-numeric key '" + + paths[0].part + "'"} + } + } + if appendit { + njson := trim(jsres.Raw) + if njson[len(njson)-1] == ']' { + njson = njson[:len(njson)-1] + } + buf = append(buf, njson...) + if comma { + buf = append(buf, ',') + } + + buf = appendBuild(buf, true, paths, raw, stringify) + buf = append(buf, ']') + return buf, nil + } + buf = append(buf, '[') + ress := jsres.Array() + for i := 0; i < len(ress); i++ { + if i > 0 { + buf = append(buf, ',') + } + buf = append(buf, ress[i].Raw...) + } + if len(ress) == 0 { + buf = appendRepeat(buf, "null,", n-len(ress)) + } else { + buf = appendRepeat(buf, ",null", n-len(ress)) + if comma { + buf = append(buf, ',') + } + } + buf = appendBuild(buf, true, paths, raw, stringify) + buf = append(buf, ']') + return buf, nil + } +} + +func isOptimisticPath(path string) bool { + for i := 0; i < len(path); i++ { + if path[i] < '.' || path[i] > 'z' { + return false + } + if path[i] > '9' && path[i] < 'A' { + return false + } + if path[i] > 'z' { + return false + } + } + return true +} + +// Set sets a json value for the specified path. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// An error is returned if the path is not valid. +// +// A path is a series of keys separated by a dot. +// +// { +// "name": {"first": "Tom", "last": "Anderson"}, +// "age":37, +// "children": ["Sara","Alex","Jack"], +// "friends": [ +// {"first": "James", "last": "Murphy"}, +// {"first": "Roger", "last": "Craig"} +// ] +// } +// "name.last" >> "Anderson" +// "age" >> 37 +// "children.1" >> "Alex" +// +func Set(json, path string, value interface{}) (string, error) { + return SetOptions(json, path, value, nil) +} + +// SetBytes sets a json value for the specified path. +// If working with bytes, this method preferred over +// Set(string(data), path, value) +func SetBytes(json []byte, path string, value interface{}) ([]byte, error) { + return SetBytesOptions(json, path, value, nil) +} + +// SetRaw sets a raw json value for the specified path. +// This function works the same as Set except that the value is set as a +// raw block of json. This allows for setting premarshalled json objects. +func SetRaw(json, path, value string) (string, error) { + return SetRawOptions(json, path, value, nil) +} + +// SetRawOptions sets a raw json value for the specified path with options. +// This furnction works the same as SetOptions except that the value is set +// as a raw block of json. This allows for setting premarshalled json objects. +func SetRawOptions(json, path, value string, opts *Options) (string, error) { + var optimistic bool + if opts != nil { + optimistic = opts.Optimistic + } + res, err := set(json, path, value, false, false, optimistic, false) + if err == errNoChange { + return json, nil + } + return string(res), err +} + +// SetRawBytes sets a raw json value for the specified path. +// If working with bytes, this method preferred over +// SetRaw(string(data), path, value) +func SetRawBytes(json []byte, path string, value []byte) ([]byte, error) { + return SetRawBytesOptions(json, path, value, nil) +} + +type dtype struct{} + +// Delete deletes a value from json for the specified path. +func Delete(json, path string) (string, error) { + return Set(json, path, dtype{}) +} + +// DeleteBytes deletes a value from json for the specified path. +func DeleteBytes(json []byte, path string) ([]byte, error) { + return SetBytes(json, path, dtype{}) +} + +type stringHeader struct { + data unsafe.Pointer + len int +} + +type sliceHeader struct { + data unsafe.Pointer + len int + cap int +} + +func set(jstr, path, raw string, + stringify, del, optimistic, inplace bool) ([]byte, error) { + if path == "" { + return []byte(jstr), &errorType{"path cannot be empty"} + } + if !del && optimistic && isOptimisticPath(path) { + res := gjson.Get(jstr, path) + if res.Exists() && res.Index > 0 { + sz := len(jstr) - len(res.Raw) + len(raw) + if stringify { + sz += 2 + } + if inplace && sz <= len(jstr) { + if !stringify || !mustMarshalString(raw) { + jsonh := *(*stringHeader)(unsafe.Pointer(&jstr)) + jsonbh := sliceHeader{ + data: jsonh.data, len: jsonh.len, cap: jsonh.len} + jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh)) + if stringify { + jbytes[res.Index] = '"' + copy(jbytes[res.Index+1:], []byte(raw)) + jbytes[res.Index+1+len(raw)] = '"' + copy(jbytes[res.Index+1+len(raw)+1:], + jbytes[res.Index+len(res.Raw):]) + } else { + copy(jbytes[res.Index:], []byte(raw)) + copy(jbytes[res.Index+len(raw):], + jbytes[res.Index+len(res.Raw):]) + } + return jbytes[:sz], nil + } + return []byte(jstr), nil + } + buf := make([]byte, 0, sz) + buf = append(buf, jstr[:res.Index]...) + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + buf = append(buf, jstr[res.Index+len(res.Raw):]...) + return buf, nil + } + } + var paths []pathResult + r, simple := parsePath(path) + if simple { + paths = append(paths, r) + for r.more { + r, simple = parsePath(r.path) + if !simple { + break + } + paths = append(paths, r) + } + } + if !simple { + if del { + return []byte(jstr), + &errorType{"cannot delete value from a complex path"} + } + return setComplexPath(jstr, path, raw, stringify) + } + njson, err := appendRawPaths(nil, jstr, paths, raw, stringify, del) + if err != nil { + return []byte(jstr), err + } + return njson, nil +} + +func setComplexPath(jstr, path, raw string, stringify bool) ([]byte, error) { + res := gjson.Get(jstr, path) + if !res.Exists() || !(res.Index != 0 || len(res.Indexes) != 0) { + return []byte(jstr), errNoChange + } + if res.Index != 0 { + njson := []byte(jstr[:res.Index]) + if stringify { + njson = appendStringify(njson, raw) + } else { + njson = append(njson, raw...) + } + njson = append(njson, jstr[res.Index+len(res.Raw):]...) + jstr = string(njson) + } + if len(res.Indexes) > 0 { + type val struct { + index int + res gjson.Result + } + vals := make([]val, 0, len(res.Indexes)) + res.ForEach(func(_, vres gjson.Result) bool { + vals = append(vals, val{res: vres}) + return true + }) + if len(res.Indexes) != len(vals) { + return []byte(jstr), errNoChange + } + for i := 0; i < len(res.Indexes); i++ { + vals[i].index = res.Indexes[i] + } + sort.SliceStable(vals, func(i, j int) bool { + return vals[i].index > vals[j].index + }) + for _, val := range vals { + vres := val.res + index := val.index + njson := []byte(jstr[:index]) + if stringify { + njson = appendStringify(njson, raw) + } else { + njson = append(njson, raw...) + } + njson = append(njson, jstr[index+len(vres.Raw):]...) + jstr = string(njson) + } + } + return []byte(jstr), nil +} + +// SetOptions sets a json value for the specified path with options. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// An error is returned if the path is not valid. +func SetOptions(json, path string, value interface{}, + opts *Options) (string, error) { + if opts != nil { + if opts.ReplaceInPlace { + // it's not safe to replace bytes in-place for strings + // copy the Options and set options.ReplaceInPlace to false. + nopts := *opts + opts = &nopts + opts.ReplaceInPlace = false + } + } + jsonh := *(*stringHeader)(unsafe.Pointer(&json)) + jsonbh := sliceHeader{data: jsonh.data, len: jsonh.len, cap: jsonh.len} + jsonb := *(*[]byte)(unsafe.Pointer(&jsonbh)) + res, err := SetBytesOptions(jsonb, path, value, opts) + return string(res), err +} + +// SetBytesOptions sets a json value for the specified path with options. +// If working with bytes, this method preferred over +// SetOptions(string(data), path, value) +func SetBytesOptions(json []byte, path string, value interface{}, + opts *Options) ([]byte, error) { + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + jstr := *(*string)(unsafe.Pointer(&json)) + var res []byte + var err error + switch v := value.(type) { + default: + b, merr := jsongo.Marshal(value) + if merr != nil { + return nil, merr + } + raw := *(*string)(unsafe.Pointer(&b)) + res, err = set(jstr, path, raw, false, false, optimistic, inplace) + case dtype: + res, err = set(jstr, path, "", false, true, optimistic, inplace) + case string: + res, err = set(jstr, path, v, true, false, optimistic, inplace) + case []byte: + raw := *(*string)(unsafe.Pointer(&v)) + res, err = set(jstr, path, raw, true, false, optimistic, inplace) + case bool: + if v { + res, err = set(jstr, path, "true", false, false, optimistic, inplace) + } else { + res, err = set(jstr, path, "false", false, false, optimistic, inplace) + } + case int8: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int16: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int32: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int64: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case uint8: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint16: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint32: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint64: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case float32: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + case float64: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + } + if err == errNoChange { + return json, nil + } + return res, err +} + +// SetRawBytesOptions sets a raw json value for the specified path with options. +// If working with bytes, this method preferred over +// SetRawOptions(string(data), path, value, opts) +func SetRawBytesOptions(json []byte, path string, value []byte, + opts *Options) ([]byte, error) { + jstr := *(*string)(unsafe.Pointer(&json)) + vstr := *(*string)(unsafe.Pointer(&value)) + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + res, err := set(jstr, path, vstr, false, false, optimistic, inplace) + if err == errNoChange { + return json, nil + } + return res, err +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6dfe0a3b7..2fa7820ce 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1830,6 +1830,9 @@ github.com/tidwall/match # github.com/tidwall/pretty v1.2.1 ## explicit; go 1.16 github.com/tidwall/pretty +# github.com/tidwall/sjson v1.2.5 +## explicit; go 1.14 +github.com/tidwall/sjson # github.com/tinylib/msgp v1.3.0 ## explicit; go 1.20 github.com/tinylib/msgp/msgp