From ef47f7a222924a3161d5e376c80c450873bb7263 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Thu, 21 Jul 2022 15:28:25 -0700 Subject: [PATCH 1/7] Remove Super Schema field from RootValue flatbuffer schema --- go/gen/fb/serial/rootvalue.go | 42 +------------ go/libraries/doltcore/doltdb/root_val.go | 79 +----------------------- go/serial/rootvalue.fbs | 1 - go/store/types/serial_message.go | 11 ---- 4 files changed, 3 insertions(+), 130 deletions(-) diff --git a/go/gen/fb/serial/rootvalue.go b/go/gen/fb/serial/rootvalue.go index 5ebfa53f51..53ea2c8f87 100644 --- a/go/gen/fb/serial/rootvalue.go +++ b/go/gen/fb/serial/rootvalue.go @@ -127,42 +127,8 @@ func (rcv *RootValue) MutateForeignKeyAddr(j int, n byte) bool { return false } -func (rcv *RootValue) SuperSchemasAddr(j int) byte { - o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) - if o != 0 { - a := rcv._tab.Vector(o) - return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1)) - } - return 0 -} - -func (rcv *RootValue) SuperSchemasAddrLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) - if o != 0 { - return rcv._tab.VectorLen(o) - } - return 0 -} - -func (rcv *RootValue) SuperSchemasAddrBytes() []byte { - o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) - if o != 0 { - return rcv._tab.ByteVector(o + rcv._tab.Pos) - } - return nil -} - -func (rcv *RootValue) MutateSuperSchemasAddr(j int, n byte) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) - if o != 0 { - a := rcv._tab.Vector(o) - return rcv._tab.MutateByte(a+flatbuffers.UOffsetT(j*1), n) - } - return false -} - func RootValueStart(builder *flatbuffers.Builder) { - builder.StartObject(4) + builder.StartObject(3) } func RootValueAddFeatureVersion(builder *flatbuffers.Builder, featureVersion int64) { builder.PrependInt64Slot(0, featureVersion, 0) @@ -179,12 +145,6 @@ func RootValueAddForeignKeyAddr(builder *flatbuffers.Builder, foreignKeyAddr fla func RootValueStartForeignKeyAddrVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(1, numElems, 1) } -func RootValueAddSuperSchemasAddr(builder *flatbuffers.Builder, superSchemasAddr flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(superSchemasAddr), 0) -} -func RootValueStartSuperSchemasAddrVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { - return builder.StartVector(1, numElems, 1) -} func RootValueEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } diff --git a/go/libraries/doltcore/doltdb/root_val.go b/go/libraries/doltcore/doltdb/root_val.go index df0e2612bb..f873ade7ff 100644 --- a/go/libraries/doltcore/doltdb/root_val.go +++ b/go/libraries/doltcore/doltdb/root_val.go @@ -83,10 +83,8 @@ type rvStorage interface { GetFeatureVersion() (FeatureVersion, bool, error) GetTablesMap(ctx context.Context, vr types.ValueReadWriter, ns tree.NodeStore) (tableMap, error) - GetSuperSchemaMap(ctx context.Context, vr types.ValueReader) (types.Map, bool, error) GetForeignKeys(ctx context.Context, vr types.ValueReader) (types.Value, bool, error) - SetSuperSchemaMap(ctx context.Context, vrw types.ValueReadWriter, m types.Map) (rvStorage, error) SetForeignKeyMap(ctx context.Context, vrw types.ValueReadWriter, m types.Value) (rvStorage, error) SetFeatureVersion(v FeatureVersion) (rvStorage, error) @@ -326,12 +324,10 @@ func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter, ns tree.Node var empty hash.Hash fkoff := builder.CreateByteVector(empty[:]) - ssoff := builder.CreateByteVector(empty[:]) serial.RootValueStart(builder) serial.RootValueAddFeatureVersion(builder, int64(DoltFeatureVersion)) serial.RootValueAddTables(builder, tablesoff) serial.RootValueAddForeignKeyAddr(builder, fkoff) - serial.RootValueAddSuperSchemasAddr(builder, ssoff) builder.FinishWithFileIdentifier(serial.RootValueEnd(builder), []byte(serial.RootValueFileID)) bs := builder.FinishedBytes() return newRootValue(vrw, ns, types.SerialMessage(bs)) @@ -500,23 +496,6 @@ func (root *RootValue) GenerateTagsForNewColumns( return newTags, nil } -// GerSuperSchemaMap returns the Noms map that tracks SuperSchemas, used to create new RootValues on checkout branch. -func (root *RootValue) GetSuperSchemaMap(ctx context.Context) (types.Map, error) { - return root.getOrCreateSuperSchemaMap(ctx) -} - -func (root *RootValue) getOrCreateSuperSchemaMap(ctx context.Context) (types.Map, error) { - m, found, err := root.st.GetSuperSchemaMap(ctx, root.vrw) - if err != nil { - return types.Map{}, err - } - if found { - return m, nil - } - - return types.NewMap(ctx, root.vrw) -} - func (root *RootValue) GetAllSchemas(ctx context.Context) (map[string]schema.Schema, error) { m := make(map[string]schema.Schema) err := root.IterTables(ctx, func(name string, table *Table, sch schema.Schema) (stop bool, err error) { @@ -839,35 +818,10 @@ func (root *RootValue) HashOf() (hash.Hash, error) { // RenameTable renames a table by changing its string key in the RootValue's table map. In order to preserve // column tag information, use this method instead of a table drop + add. func (root *RootValue) RenameTable(ctx context.Context, oldName, newName string) (*RootValue, error) { - newStorage, err := root.st.EditTablesMap(ctx, root.vrw, root.ns, []tableEdit{{old_name: oldName, name: newName}}) if err != nil { return nil, err } - - ssMap, err := root.getOrCreateSuperSchemaMap(ctx) - if err != nil { - return nil, err - } - - ssv, found, err := ssMap.MaybeGet(ctx, types.String(oldName)) - if err != nil { - return nil, err - } - if found { - ssme := ssMap.Edit().Remove(types.String(oldName)) - ssme = ssme.Set(types.String(newName), ssv) - ssMap, err = ssme.Map(ctx) - if err != nil { - return nil, err - } - - newStorage, err = newStorage.SetSuperSchemaMap(ctx, root.vrw, ssMap) - if err != nil { - return nil, err - } - } - return root.withStorage(newStorage), nil } @@ -1172,18 +1126,6 @@ func (m fbTableMap) Iter(ctx context.Context, cb func(string, hash.Hash) (bool, }) } -func (r fbRvStorage) GetSuperSchemaMap(ctx context.Context, vr types.ValueReader) (types.Map, bool, error) { - addr := hash.New(r.srv.SuperSchemasAddrBytes()) - if addr.IsEmpty() { - return types.Map{}, false, nil - } - v, err := vr.ReadValue(ctx, addr) - if err != nil { - return types.Map{}, false, err - } - return v.(types.Map), true, nil -} - func (r fbRvStorage) GetForeignKeys(ctx context.Context, vr types.ValueReader) (types.Value, bool, error) { addr := hash.New(r.srv.ForeignKeyAddrBytes()) if addr.IsEmpty() { @@ -1196,20 +1138,6 @@ func (r fbRvStorage) GetForeignKeys(ctx context.Context, vr types.ValueReader) ( return v.(types.SerialMessage), true, nil } -func (r fbRvStorage) SetSuperSchemaMap(ctx context.Context, vrw types.ValueReadWriter, m types.Map) (rvStorage, error) { - var h hash.Hash - if !m.Empty() { - ref, err := vrw.WriteValue(ctx, m) - if err != nil { - return nil, err - } - h = ref.TargetHash() - } - ret := r.clone() - copy(ret.srv.SuperSchemasAddrBytes(), h[:]) - return ret, nil -} - func (r fbRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, edits []tableEdit) (rvStorage, error) { builder := flatbuffers.NewBuilder(80) @@ -1262,12 +1190,10 @@ func (r fbRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWrite tablesoff := builder.CreateByteVector(ambytes) fkoff := builder.CreateByteVector(r.srv.ForeignKeyAddrBytes()) - ssoff := builder.CreateByteVector(r.srv.SuperSchemasAddrBytes()) serial.RootValueStart(builder) serial.RootValueAddFeatureVersion(builder, r.srv.FeatureVersion()) serial.RootValueAddTables(builder, tablesoff) serial.RootValueAddForeignKeyAddr(builder, fkoff) - serial.RootValueAddSuperSchemasAddr(builder, ssoff) builder.FinishWithFileIdentifier(serial.RootValueEnd(builder), []byte(serial.RootValueFileID)) bs := builder.FinishedBytes() return fbRvStorage{serial.GetRootAsRootValue(bs, 0)}, nil @@ -1302,11 +1228,10 @@ func (r fbRvStorage) clone() fbRvStorage { } func (r fbRvStorage) DebugString(ctx context.Context) string { - return fmt.Sprintf("fbRvStorage[%d, %s, %s, %s]", + return fmt.Sprintf("fbRvStorage[%d, %s, %s]", r.srv.FeatureVersion(), "...", // TODO: Print out tables map - hash.New(r.srv.ForeignKeyAddrBytes()).String(), - hash.New(r.srv.SuperSchemasAddrBytes()).String()) + hash.New(r.srv.ForeignKeyAddrBytes()).String()) } func (r fbRvStorage) nomsValue() types.Value { diff --git a/go/serial/rootvalue.fbs b/go/serial/rootvalue.fbs index fd4656623e..3f1b869a88 100644 --- a/go/serial/rootvalue.fbs +++ b/go/serial/rootvalue.fbs @@ -22,7 +22,6 @@ table RootValue { tables:[ubyte]; // Serialized AddressMap. foreign_key_addr:[ubyte]; - super_schemas_addr:[ubyte]; } // KEEP THIS IN SYNC WITH fileidentifiers.go diff --git a/go/store/types/serial_message.go b/go/store/types/serial_message.go index 8a9d5b875d..05f4449d1c 100644 --- a/go/store/types/serial_message.go +++ b/go/store/types/serial_message.go @@ -107,7 +107,6 @@ func (sm SerialMessage) HumanReadableString() string { fmt.Fprintf(ret, "{\n") fmt.Fprintf(ret, "\tFeatureVersion: %d\n", msg.FeatureVersion()) fmt.Fprintf(ret, "\tForeignKeys: #%s\n", hash.New(msg.ForeignKeyAddrBytes()).String()) - fmt.Fprintf(ret, "\tSuperSchema: #%s\n", hash.New(msg.SuperSchemasAddrBytes()).String()) fmt.Fprintf(ret, "\tTables: {\n\t%s", TupleRowStorage(msg.TablesBytes()).HumanReadableString()) fmt.Fprintf(ret, "\t}\n") fmt.Fprintf(ret, "}") @@ -222,16 +221,6 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { return err } } - addr = hash.New(msg.SuperSchemasAddrBytes()) - if !addr.IsEmpty() { - r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight) - if err != nil { - return err - } - if err = cb(r); err != nil { - return err - } - } case serial.TableFileID: msg := serial.GetRootAsTable([]byte(sm), 0) addr := hash.New(msg.SchemaBytes()) From 6b28b2edde6c6db142b57478a3d7c92b2932a805 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 25 Jul 2022 11:45:44 -0700 Subject: [PATCH 2/7] /.github: add readme, fix file names --- .github/README.md | 28 +++++++ ...aml => import-benchmarks-pull-report.yaml} | 0 ...chmarks.yaml => k8s-benchmark-import.yaml} | 0 ...hmarks.yaml => k8s-benchmark-latency.yaml} | 0 .github/workflows/k8s-sql-correctness.yaml | 74 ++++++++++--------- ...-performance-benchmarks-email-report.yaml} | 0 ... performance-benchmarks-email-report.yaml} | 0 ...> performance-benchmarks-pull-report.yaml} | 0 .github/workflows/sql-correctness.yaml | 74 +++++++++---------- 9 files changed, 102 insertions(+), 74 deletions(-) create mode 100644 .github/README.md rename .github/workflows/{k8s-import-benchmarks-pull-report.yaml => import-benchmarks-pull-report.yaml} (100%) rename .github/workflows/{import-benchmarks.yaml => k8s-benchmark-import.yaml} (100%) rename .github/workflows/{latency-benchmarks.yaml => k8s-benchmark-latency.yaml} (100%) rename .github/workflows/{k8s-nightly-performance-benchmarks-email-report.yaml => nightly-performance-benchmarks-email-report.yaml} (100%) rename .github/workflows/{k8s-performance-benchmarks-email-report.yaml => performance-benchmarks-email-report.yaml} (100%) rename .github/workflows/{k8s-performance-benchmarks-pull-report.yaml => performance-benchmarks-pull-report.yaml} (100%) diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000000..d6b3f6d8d8 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,28 @@ +# Dolt's GitHub Actions + +This doc will provide context for the types of Workflows we use in this repository. This doc is not a comprehensive GitHub Actions tutorial. To familiarize yourself with GitHub Actions concepts and the terminology, please see the [documentation](https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions). + +Dolt uses GitHub Actions Workflows in four primary ways: + + * To run continuous integration tests on pull requests and pushes to `main` + * To release and publish new Dolt assets + * To deploy various benchmarking jobs to contexts _other_ than GitHub Actions (like in a Kubernetes cluster, for example). + * To handle misc. [repository_dispatch](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#repository_dispatch) events triggered by external clients. + + ## Continuous Integration Workflows + +Workflows prefixed with `ci-` are run on pull requests to `main`, though some run on pushes to `main` (after a pull request is merged). These workflows are synchronous and don't trigger any other workflows to run. + +## Dolt Release Workflows + +Workflows prefixed with `cd-` are used for releasing Dolt. Some of these workflows are asynchronous, meaning that they only perform part of a task before triggering the next part of a task to run in a _different_ workflow, sometimes in other GitHub repositories, using `repository_dispatch` events. + +## Benchmarking Workflows + +Benchmarking workflows are used as an interface for deploying benchmarking jobs to one of our Kubernetes Clusters. Workflows that deploy Kubernetes Jobs are prefixed with `k8s-` and can only be triggered with `repository_dispatch` events. Notice that benchmarking workflows, like `workflows/performance-benchmarks-email-report.yaml` for example, trigger these events using the `peter-evans/repository-dispatch@v1` Action. + +These Kubernetes Jobs do not run on GitHub Actions Hosted Runners, so the workflow logs do not contain any information about the deployed Kubernetes Job or any errors it might have encountered. The workflow logs can only tell you if a Job was created successfully or not. To investigate an error or issue with a Job in our Kubernetes Cluster, see the debugging guide [here](). + +## Misc. Repository Dispatch Workflows + +Some workflows perform single, common tasks and are triggered by `repository_dispatch` events. These include the `workflows/email-report.yaml` that emails the results of performance benchmarks to the team, or the `workflows/pull-report.yaml` that posts those same results to an open pull request. Workflows like these are triggered by external clients. diff --git a/.github/workflows/k8s-import-benchmarks-pull-report.yaml b/.github/workflows/import-benchmarks-pull-report.yaml similarity index 100% rename from .github/workflows/k8s-import-benchmarks-pull-report.yaml rename to .github/workflows/import-benchmarks-pull-report.yaml diff --git a/.github/workflows/import-benchmarks.yaml b/.github/workflows/k8s-benchmark-import.yaml similarity index 100% rename from .github/workflows/import-benchmarks.yaml rename to .github/workflows/k8s-benchmark-import.yaml diff --git a/.github/workflows/latency-benchmarks.yaml b/.github/workflows/k8s-benchmark-latency.yaml similarity index 100% rename from .github/workflows/latency-benchmarks.yaml rename to .github/workflows/k8s-benchmark-latency.yaml diff --git a/.github/workflows/k8s-sql-correctness.yaml b/.github/workflows/k8s-sql-correctness.yaml index 7b7673729f..d6ad59917b 100644 --- a/.github/workflows/k8s-sql-correctness.yaml +++ b/.github/workflows/k8s-sql-correctness.yaml @@ -1,43 +1,49 @@ -name: Benchmark SQL Correctness +name: SQL Correctness on: repository_dispatch: - types: [ release-dolt ] - workflow_dispatch: - inputs: - version: - description: 'SemVer format release tag, i.e. 0.24.5' - required: true - default: '' - email: - description: 'Email address to receive results' - required: true - default: '' + types: [ sql-correctness ] jobs: - set-version-actor: - name: Set Version and Actor - runs-on: ubuntu-18.04 - outputs: - version: ${{ steps.set-vars.outputs.version }} - actor: ${{ steps.set-vars.outputs.actor }} - steps: - - name: Set variables - id: set-vars - run: | - echo "::set-output name=version::$VERSION" - echo "::set-output name=actor::$ACTOR" - env: - VERSION: ${{ github.event.inputs.version || github.event.client_payload.version }} - ACTOR: ${{ github.event.client_payload.actor || github.actor }} - correctness: runs-on: ubuntu-18.04 - needs: set-version-actor - name: Benchmark SQL Correctness + name: Dolt SQL Correctness + strategy: + matrix: + dolt_fmt: [ "__LD_1__", "__DOLT_1__" ] steps: - - uses: peter-evans/repository-dispatch@v1 + - uses: actions/checkout@v2 + - uses: azure/setup-kubectl@v2.0 with: - token: ${{ secrets.REPO_ACCESS_TOKEN }} - event-type: sql-correctness - client-payload: '{"to_version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh"}' + version: 'v1.23.6' + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Install aws-iam-authenticator + run: | + curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \ + chmod +x ./aws-iam-authenticator && \ + sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator + aws-iam-authenticator version + - name: Create and Auth kubeconfig + run: | + echo "$CONFIG" > kubeconfig + KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-dolt --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1 + KUBECONFIG=kubeconfig kubectl config set-context github-actions-dolt-context --cluster=eks-cluster-1 --user=github-actions-dolt --namespace=performance-benchmarking + KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context + env: + CONFIG: ${{ secrets.CORP_KUBECONFIG }} + - name: Run correctness + run: ./.github/scripts/sql-correctness/run-correctness.sh + env: + TO_VERSION: ${{ github.event.client_payload.to_version }} + MODE: ${{ github.event.client_payload.mode }} + ACTOR: ${{ github.event.client_payload.actor }} + ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }} + REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + KUBECONFIG: "./kubeconfig" + NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} + TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }} diff --git a/.github/workflows/k8s-nightly-performance-benchmarks-email-report.yaml b/.github/workflows/nightly-performance-benchmarks-email-report.yaml similarity index 100% rename from .github/workflows/k8s-nightly-performance-benchmarks-email-report.yaml rename to .github/workflows/nightly-performance-benchmarks-email-report.yaml diff --git a/.github/workflows/k8s-performance-benchmarks-email-report.yaml b/.github/workflows/performance-benchmarks-email-report.yaml similarity index 100% rename from .github/workflows/k8s-performance-benchmarks-email-report.yaml rename to .github/workflows/performance-benchmarks-email-report.yaml diff --git a/.github/workflows/k8s-performance-benchmarks-pull-report.yaml b/.github/workflows/performance-benchmarks-pull-report.yaml similarity index 100% rename from .github/workflows/k8s-performance-benchmarks-pull-report.yaml rename to .github/workflows/performance-benchmarks-pull-report.yaml diff --git a/.github/workflows/sql-correctness.yaml b/.github/workflows/sql-correctness.yaml index d6ad59917b..7b7673729f 100644 --- a/.github/workflows/sql-correctness.yaml +++ b/.github/workflows/sql-correctness.yaml @@ -1,49 +1,43 @@ -name: SQL Correctness +name: Benchmark SQL Correctness on: repository_dispatch: - types: [ sql-correctness ] + types: [ release-dolt ] + workflow_dispatch: + inputs: + version: + description: 'SemVer format release tag, i.e. 0.24.5' + required: true + default: '' + email: + description: 'Email address to receive results' + required: true + default: '' jobs: + set-version-actor: + name: Set Version and Actor + runs-on: ubuntu-18.04 + outputs: + version: ${{ steps.set-vars.outputs.version }} + actor: ${{ steps.set-vars.outputs.actor }} + steps: + - name: Set variables + id: set-vars + run: | + echo "::set-output name=version::$VERSION" + echo "::set-output name=actor::$ACTOR" + env: + VERSION: ${{ github.event.inputs.version || github.event.client_payload.version }} + ACTOR: ${{ github.event.client_payload.actor || github.actor }} + correctness: runs-on: ubuntu-18.04 - name: Dolt SQL Correctness - strategy: - matrix: - dolt_fmt: [ "__LD_1__", "__DOLT_1__" ] + needs: set-version-actor + name: Benchmark SQL Correctness steps: - - uses: actions/checkout@v2 - - uses: azure/setup-kubectl@v2.0 + - uses: peter-evans/repository-dispatch@v1 with: - version: 'v1.23.6' - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Install aws-iam-authenticator - run: | - curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \ - chmod +x ./aws-iam-authenticator && \ - sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator - aws-iam-authenticator version - - name: Create and Auth kubeconfig - run: | - echo "$CONFIG" > kubeconfig - KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-dolt --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1 - KUBECONFIG=kubeconfig kubectl config set-context github-actions-dolt-context --cluster=eks-cluster-1 --user=github-actions-dolt --namespace=performance-benchmarking - KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context - env: - CONFIG: ${{ secrets.CORP_KUBECONFIG }} - - name: Run correctness - run: ./.github/scripts/sql-correctness/run-correctness.sh - env: - TO_VERSION: ${{ github.event.client_payload.to_version }} - MODE: ${{ github.event.client_payload.mode }} - ACTOR: ${{ github.event.client_payload.actor }} - ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }} - REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} - KUBECONFIG: "./kubeconfig" - NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} - TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }} + token: ${{ secrets.REPO_ACCESS_TOKEN }} + event-type: sql-correctness + client-payload: '{"to_version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh"}' From a1edfb2d2461c7b701211de77aaba3655e165a45 Mon Sep 17 00:00:00 2001 From: bheni Date: Mon, 25 Jul 2022 22:19:21 +0000 Subject: [PATCH 3/7] [ga-bump-release] Update Dolt version to 0.40.20 and release v0.40.20 --- go/cmd/dolt/dolt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/cmd/dolt/dolt.go b/go/cmd/dolt/dolt.go index fbd9a76792..fa2fd6fd93 100644 --- a/go/cmd/dolt/dolt.go +++ b/go/cmd/dolt/dolt.go @@ -56,7 +56,7 @@ import ( ) const ( - Version = "0.40.19" + Version = "0.40.20" ) var dumpDocsCommand = &commands.DumpDocsCmd{} From 676c6dcd993fdd38f0db2006f115e343d3c11d9d Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 25 Jul 2022 16:29:58 -0700 Subject: [PATCH 4/7] /.github/workflows: disambiguate action titles --- .github/workflows/import-benchmarks-pull-report.yaml | 2 +- .github/workflows/k8s-benchmark-import.yaml | 2 +- .github/workflows/k8s-benchmark-latency.yaml | 4 ++-- .github/workflows/k8s-fuzzer.yaml | 2 +- .github/workflows/k8s-sql-correctness.yaml | 2 +- .../nightly-performance-benchmarks-email-report.yaml | 2 +- .github/workflows/performance-benchmarks-email-report.yaml | 2 +- .github/workflows/performance-benchmarks-pull-report.yaml | 2 +- .github/workflows/sql-correctness.yaml | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/import-benchmarks-pull-report.yaml b/.github/workflows/import-benchmarks-pull-report.yaml index db35b11c0c..4052cb6cc2 100644 --- a/.github/workflows/import-benchmarks-pull-report.yaml +++ b/.github/workflows/import-benchmarks-pull-report.yaml @@ -45,7 +45,7 @@ jobs: runs-on: ubuntu-18.04 needs: [validate-commentor, check-comments] if: ${{ needs.check-comments.outputs.benchmark == 'true' }} - name: Benchmark Import Performance + name: Trigger Benchmark Import K8s Workflow steps: - uses: xt0rted/pull-request-comment-branch@v1 id: comment-branch diff --git a/.github/workflows/k8s-benchmark-import.yaml b/.github/workflows/k8s-benchmark-import.yaml index 311f0ea5fc..5bee364d14 100644 --- a/.github/workflows/k8s-benchmark-import.yaml +++ b/.github/workflows/k8s-benchmark-import.yaml @@ -37,7 +37,7 @@ jobs: KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context env: CONFIG: ${{ secrets.CORP_KUBECONFIG }} - - name: Run Import benchmarks + - name: Create Import Benchmarking K8s Job run: ./.github/scripts/import-benchmarking/run-benchmarks.sh env: FROM_SERVER: ${{ github.event.client_payload.from_server }} diff --git a/.github/workflows/k8s-benchmark-latency.yaml b/.github/workflows/k8s-benchmark-latency.yaml index 9704f4c0a4..d35011baa4 100644 --- a/.github/workflows/k8s-benchmark-latency.yaml +++ b/.github/workflows/k8s-benchmark-latency.yaml @@ -37,7 +37,7 @@ jobs: KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context env: CONFIG: ${{ secrets.CORP_KUBECONFIG }} - - name: Run Sysbench benchmarks + - name: Create Sysbench Performance Benchmarking K8s Job run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh env: FROM_SERVER: ${{ github.event.client_payload.from_server }} @@ -53,7 +53,7 @@ jobs: INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }} NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }} - - name: Run Tpcc benchmarks + - name: Create TPCC Performance Benchmarking K8s Job run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh env: FROM_SERVER: ${{ github.event.client_payload.from_server }} diff --git a/.github/workflows/k8s-fuzzer.yaml b/.github/workflows/k8s-fuzzer.yaml index c0a42bc111..59b8282133 100644 --- a/.github/workflows/k8s-fuzzer.yaml +++ b/.github/workflows/k8s-fuzzer.yaml @@ -37,7 +37,7 @@ jobs: KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context env: CONFIG: ${{ secrets.CORP_KUBECONFIG }} - - name: Fuzzer gatekeeper + - name: Create Fuzzer (GateKeeper) K8s Job run: ./.github/scripts/fuzzer/run-fuzzer.sh env: VERSION: ${{ github.sha }} diff --git a/.github/workflows/k8s-sql-correctness.yaml b/.github/workflows/k8s-sql-correctness.yaml index d6ad59917b..d0fffffe6b 100644 --- a/.github/workflows/k8s-sql-correctness.yaml +++ b/.github/workflows/k8s-sql-correctness.yaml @@ -36,7 +36,7 @@ jobs: KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context env: CONFIG: ${{ secrets.CORP_KUBECONFIG }} - - name: Run correctness + - name: Create SQL Correctness K8s Job run: ./.github/scripts/sql-correctness/run-correctness.sh env: TO_VERSION: ${{ github.event.client_payload.to_version }} diff --git a/.github/workflows/nightly-performance-benchmarks-email-report.yaml b/.github/workflows/nightly-performance-benchmarks-email-report.yaml index b0aa3c29ef..495f77cc26 100644 --- a/.github/workflows/nightly-performance-benchmarks-email-report.yaml +++ b/.github/workflows/nightly-performance-benchmarks-email-report.yaml @@ -8,7 +8,7 @@ on: jobs: perf: runs-on: ubuntu-18.04 - name: Benchmark Latency, Correctness, and Imports + name: Trigger Benchmark Latency, Benchmark Import, and SQL Correctness K8s Workflows steps: - uses: peter-evans/repository-dispatch@v1 with: diff --git a/.github/workflows/performance-benchmarks-email-report.yaml b/.github/workflows/performance-benchmarks-email-report.yaml index d6a2cc7cc9..f1d4691112 100644 --- a/.github/workflows/performance-benchmarks-email-report.yaml +++ b/.github/workflows/performance-benchmarks-email-report.yaml @@ -37,7 +37,7 @@ jobs: benchmark-dolt-mysql: runs-on: ubuntu-18.04 needs: set-version-actor - name: Benchmark Dolt vs MySQL Latency, Benchmark Import + name: Trigger Benchmark Latency and Benchmark Import K8s Workflows steps: - uses: peter-evans/repository-dispatch@v1 with: diff --git a/.github/workflows/performance-benchmarks-pull-report.yaml b/.github/workflows/performance-benchmarks-pull-report.yaml index af86f9e764..1cf633729e 100644 --- a/.github/workflows/performance-benchmarks-pull-report.yaml +++ b/.github/workflows/performance-benchmarks-pull-report.yaml @@ -45,7 +45,7 @@ jobs: runs-on: ubuntu-18.04 needs: [validate-commentor, check-comments] if: ${{ needs.check-comments.outputs.benchmark == 'true' }} - name: Benchmark Latency + name: Trigger Benchmark Latency K8s Workflow steps: - uses: xt0rted/pull-request-comment-branch@v1 id: comment-branch diff --git a/.github/workflows/sql-correctness.yaml b/.github/workflows/sql-correctness.yaml index 7b7673729f..661d164d00 100644 --- a/.github/workflows/sql-correctness.yaml +++ b/.github/workflows/sql-correctness.yaml @@ -34,7 +34,7 @@ jobs: correctness: runs-on: ubuntu-18.04 needs: set-version-actor - name: Benchmark SQL Correctness + name: Trigger SQL Correctness K8s Workflow steps: - uses: peter-evans/repository-dispatch@v1 with: From 0582e5f80b1291c0d19338b95555f17303492824 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Mon, 25 Jul 2022 17:18:30 -0700 Subject: [PATCH 5/7] Update README.md --- .github/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/README.md b/.github/README.md index d6b3f6d8d8..c94cfd0dae 100644 --- a/.github/README.md +++ b/.github/README.md @@ -21,7 +21,7 @@ Workflows prefixed with `cd-` are used for releasing Dolt. Some of these workflo Benchmarking workflows are used as an interface for deploying benchmarking jobs to one of our Kubernetes Clusters. Workflows that deploy Kubernetes Jobs are prefixed with `k8s-` and can only be triggered with `repository_dispatch` events. Notice that benchmarking workflows, like `workflows/performance-benchmarks-email-report.yaml` for example, trigger these events using the `peter-evans/repository-dispatch@v1` Action. -These Kubernetes Jobs do not run on GitHub Actions Hosted Runners, so the workflow logs do not contain any information about the deployed Kubernetes Job or any errors it might have encountered. The workflow logs can only tell you if a Job was created successfully or not. To investigate an error or issue with a Job in our Kubernetes Cluster, see the debugging guide [here](). +These Kubernetes Jobs do not run on GitHub Actions Hosted Runners, so the workflow logs do not contain any information about the deployed Kubernetes Job or any errors it might have encountered. The workflow logs can only tell you if a Job was created successfully or not. To investigate an error or issue with a Job in our Kubernetes Cluster, see the debugging guide [here](https://github.com/dolthub/ld/blob/main/k8s/README.md#debug-performance-benchmarks-and-sql-correctness-jobs). ## Misc. Repository Dispatch Workflows From 9dae75ed7d9b41ff02840cd5654cc808ce51e8eb Mon Sep 17 00:00:00 2001 From: Maximilian Hoffman Date: Mon, 25 Jul 2022 18:02:14 -0700 Subject: [PATCH 6/7] Better read only error messages (#3920) * Better read only error messages * GMS bump * block table import for server lock --- go/cmd/dolt/commands/engine/sqlengine.go | 21 ++++++++++--------- go/cmd/dolt/commands/sqlserver/server.go | 7 +------ go/go.mod | 2 +- go/go.sum | 4 ++-- .../doltcore/mvdata/engine_table_writer.go | 4 ++++ integration-tests/bats/sql-server.bats | 18 +++++++++++++--- 6 files changed, 34 insertions(+), 22 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 217662e98c..a1f89b712a 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -48,14 +48,15 @@ type SqlEngine struct { } type SqlEngineConfig struct { - InitialDb string - IsReadOnly bool - PrivFilePath string - ServerUser string - ServerPass string - Autocommit bool - Bulk bool - JwksConfig []JwksConfig + InitialDb string + IsReadOnly bool + IsServerLocked bool + PrivFilePath string + ServerUser string + ServerPass string + Autocommit bool + Bulk bool + JwksConfig []JwksConfig } // NewSqlEngine returns a SqlEngine @@ -67,7 +68,7 @@ func NewSqlEngine( ) (*SqlEngine, error) { if ok, _ := mrEnv.IsLocked(); ok { - config.IsReadOnly = true + config.IsServerLocked = true } parallelism := runtime.GOMAXPROCS(0) @@ -106,7 +107,7 @@ func NewSqlEngine( } // Set up engine - engine := gms.New(analyzer.NewBuilder(pro).WithParallelism(parallelism).Build(), &gms.Config{IsReadOnly: config.IsReadOnly, TemporaryUsers: tempUsers}).WithBackgroundThreads(bThreads) + engine := gms.New(analyzer.NewBuilder(pro).WithParallelism(parallelism).Build(), &gms.Config{IsReadOnly: config.IsReadOnly, TemporaryUsers: tempUsers, IsServerLocked: config.IsServerLocked}).WithBackgroundThreads(bThreads) engine.Analyzer.Catalog.MySQLDb.SetPersister(persister) engine.Analyzer.Catalog.MySQLDb.SetPlugins(map[string]mysql_db.PlaintextAuthPlugin{ "authentication_dolt_jwt": NewAuthenticateDoltJWTPlugin(config.JwksConfig), diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index 62b434ddec..f8c7290021 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -79,11 +79,6 @@ func Serve( } logrus.SetFormatter(LogFormat{}) - isReadOnly := false - if serverConfig.ReadOnly() { - isReadOnly = true - } - var mrEnv *env.MultiRepoEnv var err error fs := dEnv.FS @@ -134,7 +129,7 @@ func Serve( // Create SQL Engine with users config := &engine.SqlEngineConfig{ InitialDb: "", - IsReadOnly: isReadOnly, + IsReadOnly: serverConfig.ReadOnly(), PrivFilePath: serverConfig.PrivilegeFilePath(), ServerUser: serverConfig.User(), ServerPass: serverConfig.Password(), diff --git a/go/go.mod b/go/go.mod index de3857843b..cce02c12bf 100644 --- a/go/go.mod +++ b/go/go.mod @@ -58,7 +58,7 @@ require ( ) require ( - github.com/dolthub/go-mysql-server v0.12.1-0.20220725181120-4bb5cae174bc + github.com/dolthub/go-mysql-server v0.12.1-0.20220725212100-c3c1731b2a03 github.com/google/flatbuffers v2.0.6+incompatible github.com/gosuri/uilive v0.0.4 github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 diff --git a/go/go.sum b/go/go.sum index 7a083be938..39f82dc88e 100644 --- a/go/go.sum +++ b/go/go.sum @@ -173,8 +173,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.12.1-0.20220725181120-4bb5cae174bc h1:IJDBkqOLPI6CGPG1oQef5QSeOEuX1eQOtO1B1f7tfMs= -github.com/dolthub/go-mysql-server v0.12.1-0.20220725181120-4bb5cae174bc/go.mod h1:JgB3WpY0RMgyAda3YG5VHVncH2B8i1N9Mx9LOp41lIs= +github.com/dolthub/go-mysql-server v0.12.1-0.20220725212100-c3c1731b2a03 h1:ek99hhyjnf3CexTF8Y5Z4XyeOlzQgMlxvFrLKEroNiU= +github.com/dolthub/go-mysql-server v0.12.1-0.20220725212100-c3c1731b2a03/go.mod h1:JgB3WpY0RMgyAda3YG5VHVncH2B8i1N9Mx9LOp41lIs= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= diff --git a/go/libraries/doltcore/mvdata/engine_table_writer.go b/go/libraries/doltcore/mvdata/engine_table_writer.go index 97adc93d31..f7b58aeab3 100644 --- a/go/libraries/doltcore/mvdata/engine_table_writer.go +++ b/go/libraries/doltcore/mvdata/engine_table_writer.go @@ -65,6 +65,10 @@ type SqlEngineTableWriter struct { } func NewSqlEngineTableWriter(ctx context.Context, dEnv *env.DoltEnv, createTableSchema, rowOperationSchema schema.Schema, options *MoverOptions, statsCB noms.StatsCB) (*SqlEngineTableWriter, error) { + if dEnv.IsLocked() { + return nil, env.ErrActiveServerLock.New(dEnv.LockFile()) + } + mrEnv, err := env.MultiEnvForDirectory(ctx, dEnv.Config.WriteableConfig(), dEnv.FS, dEnv.Version, dEnv.IgnoreLockFile, dEnv) if err != nil { return nil, err diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index 10f3fdb112..c04ab10860 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -167,7 +167,7 @@ SQL # attempt to create table (autocommit on), expect either some exception server_query repo1 1 "CREATE TABLE i_should_not_exist ( c0 INT - )" "" "not authorized" + )" "" "database server is set to read only mode" # Expect that there are still no tables run dolt ls @@ -205,7 +205,7 @@ SQL # make a dolt_commit query skip "read-only flag does not prevent dolt_commit" - server_query repo1 1 "select dolt_commit('--allow-empty', '-m', 'msg')" "" "not authorized: user does not have permission: write" + server_query repo1 1 "select dolt_commit('--allow-empty', '-m', 'msg')" "" "database server is set to read only mode: user does not have permission: write" } @test "sql-server: test command line modification" { @@ -1425,4 +1425,16 @@ databases: let PORT="$$ % (65536-1024) + 1024" run dolt sql-server -P $PORT [ "$status" -eq 1 ] -} \ No newline at end of file +} + +@test "sql-server: sql-server locks database to writes" { + cd repo2 + dolt sql -q "create table a (x int primary key)" + start_sql_server + run dolt sql -q "create table b (x int primary key)" + [ "$status" -eq 1 ] + [[ "$output" =~ "database is locked to writes" ]] || false + run dolt sql -q "insert into b values (0)" + [ "$status" -eq 1 ] + [[ "$output" =~ "database is locked to writes" ]] || false +} From 173a3ddbd4e4b544a6ef22eea6128acce360dbcb Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Mon, 25 Jul 2022 21:28:37 -0700 Subject: [PATCH 7/7] [ga-bump-dep] Bump dependency in Dolt by jennifersp (#3926) Co-authored-by: jennifersp <44716627+jennifersp@users.noreply.github.com> --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index cce02c12bf..e01ce89078 100644 --- a/go/go.mod +++ b/go/go.mod @@ -58,7 +58,7 @@ require ( ) require ( - github.com/dolthub/go-mysql-server v0.12.1-0.20220725212100-c3c1731b2a03 + github.com/dolthub/go-mysql-server v0.12.1-0.20220726021412-60dcca3a24be github.com/google/flatbuffers v2.0.6+incompatible github.com/gosuri/uilive v0.0.4 github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 diff --git a/go/go.sum b/go/go.sum index 39f82dc88e..bb7ea3eb43 100644 --- a/go/go.sum +++ b/go/go.sum @@ -173,8 +173,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.12.1-0.20220725212100-c3c1731b2a03 h1:ek99hhyjnf3CexTF8Y5Z4XyeOlzQgMlxvFrLKEroNiU= -github.com/dolthub/go-mysql-server v0.12.1-0.20220725212100-c3c1731b2a03/go.mod h1:JgB3WpY0RMgyAda3YG5VHVncH2B8i1N9Mx9LOp41lIs= +github.com/dolthub/go-mysql-server v0.12.1-0.20220726021412-60dcca3a24be h1:0nkOM5NJjD7GGVSJm+EtWghopFRKURsy0ahLv8nA84Q= +github.com/dolthub/go-mysql-server v0.12.1-0.20220726021412-60dcca3a24be/go.mod h1:JgB3WpY0RMgyAda3YG5VHVncH2B8i1N9Mx9LOp41lIs= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8=