From bf8d3e8f4d0443e1566ea31b47b1683a57499a0c Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Mon, 23 Sep 2024 13:49:58 -0700 Subject: [PATCH 01/14] [ga-bump-dep] Bump dependency in Dolt by jycor (#8375) Co-authored-by: jycor --- go/go.mod | 4 ++-- go/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go/go.mod b/go/go.mod index ea636e8252..94fe0c7617 100644 --- a/go/go.mod +++ b/go/go.mod @@ -15,7 +15,7 @@ require ( github.com/dolthub/fslock v0.0.3 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 - github.com/dolthub/vitess v0.0.0-20240916204416-9d4d4a09b1d9 + github.com/dolthub/vitess v0.0.0-20240919225659-2ad81685e772 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 @@ -57,7 +57,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.18.2-0.20240919175427-5055c7114c8b + github.com/dolthub/go-mysql-server v0.18.2-0.20240923181307-5aacdb13e45a github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 diff --git a/go/go.sum b/go/go.sum index 69cb4876ee..fd39182113 100644 --- a/go/go.sum +++ b/go/go.sum @@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662 h1:aC17hZD6iwzBwwfO5M+3oBT5E5gGRiQPdn+vzpDXqIA= github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.18.2-0.20240919175427-5055c7114c8b h1:y5oKakoZEsTZCDu67VOD395uhKG00VpVGzSYTZoYiDY= -github.com/dolthub/go-mysql-server v0.18.2-0.20240919175427-5055c7114c8b/go.mod h1:m88EMm9OthVVa6qIhbpnRDpj/eYUXuNpvY/+0YWKVwc= +github.com/dolthub/go-mysql-server v0.18.2-0.20240923181307-5aacdb13e45a h1:rpCmZj332eiBbzsHsq3Sj5AWzl3Q7szDObwI49UqA8Y= +github.com/dolthub/go-mysql-server v0.18.2-0.20240923181307-5aacdb13e45a/go.mod h1:lGbU2bK+QNnlETdUjOOaE+UnlEUu31VaQOFKAFGyZN4= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE= @@ -197,8 +197,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc= github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ= -github.com/dolthub/vitess v0.0.0-20240916204416-9d4d4a09b1d9 h1:2My8cED5m5/sFay7U4bvLxpECJccKj0cEKCqEA+63yU= -github.com/dolthub/vitess v0.0.0-20240916204416-9d4d4a09b1d9/go.mod h1:uBvlRluuL+SbEWTCZ68o0xvsdYZER3CEG/35INdzfJM= +github.com/dolthub/vitess v0.0.0-20240919225659-2ad81685e772 h1:vDwBX7Lc8DnA8Zk0iRIu6slCw0GIUfYfFlYDYJQw8GQ= +github.com/dolthub/vitess v0.0.0-20240919225659-2ad81685e772/go.mod h1:uBvlRluuL+SbEWTCZ68o0xvsdYZER3CEG/35INdzfJM= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= From e19c58a7fc038425ec3b4067ede471a1eea525fc Mon Sep 17 00:00:00 2001 From: zachmu Date: Mon, 23 Sep 2024 23:54:19 +0000 Subject: [PATCH 02/14] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/dprocedures/dolt_reset.go | 2 +- .../doltcore/sqle/dtables/constraint_violations_prolly.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_reset.go b/go/libraries/doltcore/sqle/dprocedures/dolt_reset.go index 023d654773..42c61b48e0 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_reset.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_reset.go @@ -17,7 +17,6 @@ package dprocedures import ( "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/dolt/go/cmd/dolt/cli" @@ -26,6 +25,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" "github.com/dolthub/dolt/go/libraries/utils/argparser" ) diff --git a/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go b/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go index ff91d86a01..cc3745a018 100644 --- a/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go +++ b/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go @@ -17,7 +17,6 @@ package dtables import ( "encoding/json" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" @@ -25,6 +24,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/merge" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/pool" From a27cc9bfd691a9b4ddde75f387a8f295d68bc01b Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Tue, 24 Sep 2024 06:48:04 -0700 Subject: [PATCH 03/14] go: doltcore/remotestorage: reliable: Fix leaked goroutines when the context is canceled while reading the HTTP response. io.Copy into an io.PipeWriter will block until all the bytes have been delivered or the reader is closed. reliable/http StreamingResponse was constructed to only cancel the request context on Close(), not also clear the Reader. The Reader should also be closed to ensure all finalization can still happen if the Write to the PipeWriter is currently blocked when the context is canceled. --- .../doltcore/remotestorage/internal/reliable/http.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/remotestorage/internal/reliable/http.go b/go/libraries/doltcore/remotestorage/internal/reliable/http.go index a3a4827692..c409669986 100644 --- a/go/libraries/doltcore/remotestorage/internal/reliable/http.go +++ b/go/libraries/doltcore/remotestorage/internal/reliable/http.go @@ -222,7 +222,10 @@ func StreamingRangeDownload(ctx context.Context, req StreamingRangeRequest) Stre return StreamingResponse{ Body: r, - cancel: cancel, + cancel: func() { + r.Close() + cancel() + }, } } From 4a6db7e15c902179406ab634a5a0cf12cb50c982 Mon Sep 17 00:00:00 2001 From: reltuk Date: Tue, 24 Sep 2024 13:59:21 +0000 Subject: [PATCH 04/14] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/remotestorage/internal/reliable/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/remotestorage/internal/reliable/http.go b/go/libraries/doltcore/remotestorage/internal/reliable/http.go index c409669986..cebb200afb 100644 --- a/go/libraries/doltcore/remotestorage/internal/reliable/http.go +++ b/go/libraries/doltcore/remotestorage/internal/reliable/http.go @@ -221,7 +221,7 @@ func StreamingRangeDownload(ctx context.Context, req StreamingRangeRequest) Stre }() return StreamingResponse{ - Body: r, + Body: r, cancel: func() { r.Close() cancel() From 90cddbdf56577c7697ce717518184bf1dfe7134c Mon Sep 17 00:00:00 2001 From: Taylor Bantle Date: Tue, 24 Sep 2024 11:32:51 -0700 Subject: [PATCH 05/14] Fix dolt_clean for doltgres --- go/libraries/doltcore/env/actions/reset.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/go/libraries/doltcore/env/actions/reset.go b/go/libraries/doltcore/env/actions/reset.go index bec0a40027..ede2fca413 100644 --- a/go/libraries/doltcore/env/actions/reset.go +++ b/go/libraries/doltcore/env/actions/reset.go @@ -25,6 +25,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" "github.com/dolthub/dolt/go/store/datas" ) @@ -271,7 +272,7 @@ func getUnionedTables(ctx context.Context, tables []doltdb.TableName, stagedRoot // CleanUntracked deletes untracked tables from the working root. // Evaluates untracked tables as: all working tables - all staged tables. -func CleanUntracked(ctx context.Context, roots doltdb.Roots, tables []string, dryrun bool, force bool) (doltdb.Roots, error) { +func CleanUntracked(ctx *sql.Context, roots doltdb.Roots, tables []string, dryrun bool, force bool) (doltdb.Roots, error) { untrackedTables := make(map[doltdb.TableName]struct{}) var err error @@ -284,21 +285,21 @@ func CleanUntracked(ctx context.Context, roots doltdb.Roots, tables []string, dr for i := range tables { name := tables[i] - _, _, err = roots.Working.GetTable(ctx, doltdb.TableName{Name: name}) + resolvedName, _, _, err := resolve.Table(ctx, roots.Working, name) if err != nil { return doltdb.Roots{}, err } - untrackedTables[doltdb.TableName{Name: name}] = struct{}{} + untrackedTables[resolvedName] = struct{}{} } // untracked tables = working tables - staged tables - headTblNames, err := roots.Staged.GetTableNames(ctx, doltdb.DefaultSchemaName) + headTblNames := GetAllTableNames(ctx, roots.Staged) if err != nil { return doltdb.Roots{}, err } for _, name := range headTblNames { - delete(untrackedTables, doltdb.TableName{Name: name}) + delete(untrackedTables, name) } newRoot := roots.Working From 38977f6a99aa798b52719ed1fa2fc45803288324 Mon Sep 17 00:00:00 2001 From: Taylor Bantle Date: Tue, 24 Sep 2024 11:46:54 -0700 Subject: [PATCH 06/14] Fix dolt_checkout(table) for doltgres --- .../doltcore/sqle/dprocedures/dolt_checkout.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go b/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go index 6a02fa4e92..b2600a8c80 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go @@ -28,6 +28,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/store/hash" ) @@ -476,9 +477,17 @@ func doGlobalCheckout(ctx *sql.Context, branchName string, isForce bool, isNewBr } func checkoutTables(ctx *sql.Context, roots doltdb.Roots, name string, tables []string) error { - // TODO: schema name - roots, err := actions.MoveTablesFromHeadToWorking(ctx, roots, doltdb.ToTableNames(tables, doltdb.DefaultSchemaName)) + var err error + tableNames := make([]doltdb.TableName, len(tables)) + for i, table := range tables { + tableNames[i], _, _, err = resolve.Table(ctx, roots.Working, table) + if err != nil { + return err + } + } + + roots, err = actions.MoveTablesFromHeadToWorking(ctx, roots, tableNames) if err != nil { if doltdb.IsRootValUnreachable(err) { rt := doltdb.GetUnreachableRootType(err) From fe058080deb24587a37a6e0a74f5dc7544507ce4 Mon Sep 17 00:00:00 2001 From: Taylor Bantle Date: Tue, 24 Sep 2024 12:30:00 -0700 Subject: [PATCH 07/14] Check for table in all roots for dolt_checkout --- go/libraries/doltcore/env/actions/checkout.go | 31 +++++++++++++++++++ .../sqle/dprocedures/dolt_checkout.go | 10 +++--- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/go/libraries/doltcore/env/actions/checkout.go b/go/libraries/doltcore/env/actions/checkout.go index 23bab107bd..9e83035659 100644 --- a/go/libraries/doltcore/env/actions/checkout.go +++ b/go/libraries/doltcore/env/actions/checkout.go @@ -24,6 +24,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" "github.com/dolthub/dolt/go/libraries/utils/set" "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/hash" @@ -88,6 +89,36 @@ func MoveTablesFromHeadToWorking(ctx context.Context, roots doltdb.Roots, tbls [ return roots, nil } +// FindTableInRoots resolves a table by looking in all three roots (working, +// staged, head) in that order. +func FindTableInRoots(ctx *sql.Context, roots doltdb.Roots, name string) (doltdb.TableName, *doltdb.Table, bool, error) { + tbl, root, tblExists, err := resolve.Table(ctx, roots.Working, name) + if err != nil { + return doltdb.TableName{}, nil, false, err + } + if tblExists { + return tbl, root, true, nil + } + + tbl, root, tblExists, err = resolve.Table(ctx, roots.Staged, name) + if err != nil { + return doltdb.TableName{}, nil, false, err + } + if tblExists { + return tbl, root, true, nil + } + + tbl, root, tblExists, err = resolve.Table(ctx, roots.Head, name) + if err != nil { + return doltdb.TableName{}, nil, false, err + } + if tblExists { + return tbl, root, true, nil + } + + return doltdb.TableName{}, nil, false, nil +} + // RootsForBranch returns the roots needed for a branch checkout. |roots.Head| should be the pre-checkout head. The // returned roots struct has |Head| set to |branchRoot|. func RootsForBranch(ctx context.Context, roots doltdb.Roots, branchRoot doltdb.RootValue, force bool) (doltdb.Roots, error) { diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go b/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go index b2600a8c80..097a9f54ff 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go @@ -28,7 +28,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/store/hash" ) @@ -477,17 +476,20 @@ func doGlobalCheckout(ctx *sql.Context, branchName string, isForce bool, isNewBr } func checkoutTables(ctx *sql.Context, roots doltdb.Roots, name string, tables []string) error { - var err error tableNames := make([]doltdb.TableName, len(tables)) for i, table := range tables { - tableNames[i], _, _, err = resolve.Table(ctx, roots.Working, table) + tbl, _, exists, err := actions.FindTableInRoots(ctx, roots, table) if err != nil { return err } + if !exists { + return fmt.Errorf("%w: '%s'", doltdb.ErrTableNotFound, table) + } + tableNames[i] = tbl } - roots, err = actions.MoveTablesFromHeadToWorking(ctx, roots, tableNames) + roots, err := actions.MoveTablesFromHeadToWorking(ctx, roots, tableNames) if err != nil { if doltdb.IsRootValUnreachable(err) { rt := doltdb.GetUnreachableRootType(err) From 710d4cb612d967ee3badf63bf0a9ef3371814663 Mon Sep 17 00:00:00 2001 From: Taylor Bantle Date: Tue, 24 Sep 2024 12:33:19 -0700 Subject: [PATCH 08/14] Fix bats --- go/libraries/doltcore/env/actions/reset.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/env/actions/reset.go b/go/libraries/doltcore/env/actions/reset.go index ede2fca413..18fbf3c96f 100644 --- a/go/libraries/doltcore/env/actions/reset.go +++ b/go/libraries/doltcore/env/actions/reset.go @@ -285,10 +285,13 @@ func CleanUntracked(ctx *sql.Context, roots doltdb.Roots, tables []string, dryru for i := range tables { name := tables[i] - resolvedName, _, _, err := resolve.Table(ctx, roots.Working, name) + resolvedName, _, tblExists, err := resolve.Table(ctx, roots.Working, name) if err != nil { return doltdb.Roots{}, err } + if !tblExists { + return doltdb.Roots{}, fmt.Errorf("%w: '%s'", doltdb.ErrTableNotFound, name) + } untrackedTables[resolvedName] = struct{}{} } From be6d616a50a15a0b2aabeba67b90884a71c3838d Mon Sep 17 00:00:00 2001 From: Taylor Bantle Date: Tue, 24 Sep 2024 12:51:34 -0700 Subject: [PATCH 09/14] Fix error --- go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go b/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go index 097a9f54ff..57096997db 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go @@ -484,7 +484,7 @@ func checkoutTables(ctx *sql.Context, roots doltdb.Roots, name string, tables [] return err } if !exists { - return fmt.Errorf("%w: '%s'", doltdb.ErrTableNotFound, table) + return fmt.Errorf("error: given tables do not exist") } tableNames[i] = tbl } From 7ac0e8e6b7504770d262edf1586f442730d5765c Mon Sep 17 00:00:00 2001 From: Taylor Bantle Date: Tue, 24 Sep 2024 12:20:36 -0700 Subject: [PATCH 10/14] Fix dolt_diff_stat for doltgres --- .../sqle/dolt_diff_stat_table_function.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/go/libraries/doltcore/sqle/dolt_diff_stat_table_function.go b/go/libraries/doltcore/sqle/dolt_diff_stat_table_function.go index ef31574986..8897837aa3 100644 --- a/go/libraries/doltcore/sqle/dolt_diff_stat_table_function.go +++ b/go/libraries/doltcore/sqle/dolt_diff_stat_table_function.go @@ -293,7 +293,11 @@ func (ds *DiffStatTableFunction) RowIter(ctx *sql.Context, row sql.Row) (sql.Row // If tableNameExpr defined, return a single table diff stat result if ds.tableNameExpr != nil { delta := findMatchingDelta(deltas, tableName) - diffStat, hasDiff, err := getDiffStatNodeFromDelta(ctx, delta, fromRefDetails.root, toRefDetails.root, tableName) + schemaName := delta.FromName.Schema + if schemaName == "" { + schemaName = delta.ToName.Schema + } + diffStat, hasDiff, err := getDiffStatNodeFromDelta(ctx, delta, fromRefDetails.root, toRefDetails.root, doltdb.TableName{Name: tableName, Schema: schemaName}) if err != nil { return nil, err } @@ -309,8 +313,7 @@ func (ds *DiffStatTableFunction) RowIter(ctx *sql.Context, row sql.Row) (sql.Row if tblName.Name == "" { tblName = delta.FromName } - // TODO: schema name - diffStat, hasDiff, err := getDiffStatNodeFromDelta(ctx, delta, fromRefDetails.root, toRefDetails.root, tblName.Name) + diffStat, hasDiff, err := getDiffStatNodeFromDelta(ctx, delta, fromRefDetails.root, toRefDetails.root, tblName) if err != nil { if errors.Is(err, diff.ErrPrimaryKeySetChanged) { ctx.Warn(dtables.PrimaryKeyChangeWarningCode, fmt.Sprintf("stat for table %s cannot be determined. Primary key set changed.", tblName)) @@ -369,10 +372,10 @@ func (ds *DiffStatTableFunction) evaluateArguments() (interface{}, interface{}, // getDiffStatNodeFromDelta returns diffStatNode object and whether there is data diff or not. It gets tables // from roots and diff stat if there is a valid table exists in both fromRoot and toRoot. -func getDiffStatNodeFromDelta(ctx *sql.Context, delta diff.TableDelta, fromRoot, toRoot doltdb.RootValue, tableName string) (diffStatNode, bool, error) { +func getDiffStatNodeFromDelta(ctx *sql.Context, delta diff.TableDelta, fromRoot, toRoot doltdb.RootValue, tableName doltdb.TableName) (diffStatNode, bool, error) { var oldColLen int var newColLen int - fromTable, _, fromTableExists, err := doltdb.GetTableInsensitive(ctx, fromRoot, doltdb.TableName{Name: tableName}) + fromTable, _, fromTableExists, err := doltdb.GetTableInsensitive(ctx, fromRoot, tableName) if err != nil { return diffStatNode{}, false, err } @@ -385,7 +388,7 @@ func getDiffStatNodeFromDelta(ctx *sql.Context, delta diff.TableDelta, fromRoot, oldColLen = len(fromSch.GetAllCols().GetColumns()) } - toTable, _, toTableExists, err := doltdb.GetTableInsensitive(ctx, toRoot, doltdb.TableName{Name: tableName}) + toTable, _, toTableExists, err := doltdb.GetTableInsensitive(ctx, toRoot, tableName) if err != nil { return diffStatNode{}, false, err } @@ -412,7 +415,7 @@ func getDiffStatNodeFromDelta(ctx *sql.Context, delta diff.TableDelta, fromRoot, return diffStatNode{}, false, err } - return diffStatNode{tableName, diffStat, oldColLen, newColLen, keyless}, hasDiff, nil + return diffStatNode{tableName.Name, diffStat, oldColLen, newColLen, keyless}, hasDiff, nil } // getDiffStat returns diff.DiffStatProgress object and whether there is a data diff or not. From d02b67941540435123d5bd4b69fb3508fe180849 Mon Sep 17 00:00:00 2001 From: Taylor Bantle Date: Tue, 24 Sep 2024 13:53:33 -0700 Subject: [PATCH 11/14] Fix dolt_patch and dolt_schema_diff for doltgres --- go/cmd/dolt/commands/schcmds/export.go | 2 +- go/cmd/dolt/commands/schcmds/import.go | 2 +- go/cmd/dolt/commands/schcmds/show.go | 2 +- .../doltcore/sqle/dolt_patch_table_function.go | 6 +++--- .../doltcore/sqle/dolt_schema_diff_table_function.go | 10 +++------- go/libraries/doltcore/sqle/show_create_table.go | 5 +++-- .../table/untyped/sqlexport/batch_sqlwriter.go | 2 +- .../doltcore/table/untyped/sqlexport/sqlwriter.go | 2 +- 8 files changed, 14 insertions(+), 17 deletions(-) diff --git a/go/cmd/dolt/commands/schcmds/export.go b/go/cmd/dolt/commands/schcmds/export.go index 300f1deb4e..f5af2f6c8c 100644 --- a/go/cmd/dolt/commands/schcmds/export.go +++ b/go/cmd/dolt/commands/schcmds/export.go @@ -151,7 +151,7 @@ func exportSchemas(ctx context.Context, apr *argparser.ArgParseResults, root dol func exportTblSchema(ctx context.Context, tblName string, root doltdb.RootValue, wr io.Writer, opts editor.Options) errhand.VerboseError { sqlCtx, engine, _ := dsqle.PrepareCreateTableStmt(ctx, dsqle.NewUserSpaceDatabase(root, opts)) - stmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, tblName) + stmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, doltdb.TableName{Name: tblName}) if err != nil { return errhand.VerboseErrorFromError(err) } diff --git a/go/cmd/dolt/commands/schcmds/import.go b/go/cmd/dolt/commands/schcmds/import.go index 3937bc41d7..43be2127ca 100644 --- a/go/cmd/dolt/commands/schcmds/import.go +++ b/go/cmd/dolt/commands/schcmds/import.go @@ -306,7 +306,7 @@ func importSchema(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPars sqlDb := sqle.NewUserSpaceDatabase(root, editor.Options{}) sqlCtx, engine, _ := sqle.PrepareCreateTableStmt(ctx, sqlDb) - stmt, err := sqle.GetCreateTableStmt(sqlCtx, engine, tblName) + stmt, err := sqle.GetCreateTableStmt(sqlCtx, engine, doltdb.TableName{Name: tblName}) if err != nil { return errhand.VerboseErrorFromError(err) } diff --git a/go/cmd/dolt/commands/schcmds/show.go b/go/cmd/dolt/commands/schcmds/show.go index 4e8824123f..b56d1da024 100644 --- a/go/cmd/dolt/commands/schcmds/show.go +++ b/go/cmd/dolt/commands/schcmds/show.go @@ -154,7 +154,7 @@ func printSchemas(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env notFound = append(notFound, tblName) } else { cli.Println(bold.Sprint(tblName), "@", cmStr) - stmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, tblName) + stmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, doltdb.TableName{Name: tblName}) if err != nil { return errhand.VerboseErrorFromError(err) } diff --git a/go/libraries/doltcore/sqle/dolt_patch_table_function.go b/go/libraries/doltcore/sqle/dolt_patch_table_function.go index 1a6732e60d..cd6b712b76 100644 --- a/go/libraries/doltcore/sqle/dolt_patch_table_function.go +++ b/go/libraries/doltcore/sqle/dolt_patch_table_function.go @@ -29,12 +29,12 @@ import ( "golang.org/x/exp/slices" "github.com/dolthub/dolt/go/libraries/doltcore/diff" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlfmt" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/store/types" @@ -145,11 +145,11 @@ func (p *PatchTableFunction) PartitionRows(ctx *sql.Context, partition sql.Parti // If tableNameExpr defined, return a single table patch result if p.tableNameExpr != nil { - fromTblExists, err := fromRefDetails.root.HasTable(ctx, doltdb.TableName{Name: tableName}) + _, _, fromTblExists, err := resolve.Table(ctx, fromRefDetails.root, tableName) if err != nil { return nil, err } - toTblExists, err := toRefDetails.root.HasTable(ctx, doltdb.TableName{Name: tableName}) + _, _, toTblExists, err := resolve.Table(ctx, toRefDetails.root, tableName) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/dolt_schema_diff_table_function.go b/go/libraries/doltcore/sqle/dolt_schema_diff_table_function.go index 2cb72393b0..71d85b3043 100644 --- a/go/libraries/doltcore/sqle/dolt_schema_diff_table_function.go +++ b/go/libraries/doltcore/sqle/dolt_schema_diff_table_function.go @@ -27,7 +27,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/diff" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" - "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlfmt" ) const schemaDiffDefaultRowCount = 100 @@ -314,18 +314,14 @@ func (ds *SchemaDiffTableFunction) RowIter(ctx *sql.Context, row sql.Row) (sql.R var fromCreate, toCreate string if delta.FromTable != nil { - fromSqlDb := NewUserSpaceDatabase(fromRoot, editor.Options{}) - fromSqlCtx, fromEngine, _ := PrepareCreateTableStmt(ctx, fromSqlDb) - fromCreate, err = GetCreateTableStmt(fromSqlCtx, fromEngine, delta.FromName.Name) + fromCreate, err = sqlfmt.GenerateCreateTableStatement(delta.FromName.Name, delta.FromSch, delta.FromFks, delta.FromFksParentSch) if err != nil { return nil, err } } if delta.ToTable != nil { - toSqlDb := NewUserSpaceDatabase(toRoot, editor.Options{}) - toSqlCtx, toEngine, _ := PrepareCreateTableStmt(ctx, toSqlDb) - toCreate, err = GetCreateTableStmt(toSqlCtx, toEngine, delta.ToName.Name) + toCreate, err = sqlfmt.GenerateCreateTableStatement(delta.ToName.Name, delta.ToSch, delta.ToFks, delta.ToFksParentSch) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/show_create_table.go b/go/libraries/doltcore/sqle/show_create_table.go index deab24085c..0d6d0103be 100644 --- a/go/libraries/doltcore/sqle/show_create_table.go +++ b/go/libraries/doltcore/sqle/show_create_table.go @@ -21,6 +21,7 @@ import ( sqle "github.com/dolthub/go-mysql-server" "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer" @@ -41,8 +42,8 @@ func PrepareCreateTableStmt(ctx context.Context, sqlDb dsess.SqlDatabase) (*sql. return sqlCtx, engine, sess } -func GetCreateTableStmt(ctx *sql.Context, engine *sqle.Engine, tableName string) (string, error) { - _, rowIter, _, err := engine.Query(ctx, fmt.Sprintf("SHOW CREATE TABLE `%s`;", tableName)) +func GetCreateTableStmt(ctx *sql.Context, engine *sqle.Engine, tableName doltdb.TableName) (string, error) { + _, rowIter, _, err := engine.Query(ctx, fmt.Sprintf("SHOW CREATE TABLE `%s`;", tableName.String())) if err != nil { return "", err } diff --git a/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go b/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go index 32aca3c252..f4c4617c4f 100644 --- a/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go +++ b/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go @@ -154,7 +154,7 @@ func (w *BatchSqlExportWriter) maybeWriteDropCreate(ctx context.Context) error { b.WriteString(sqlfmt.DropTableIfExistsStmt(w.tableName)) b.WriteRune('\n') sqlCtx, engine, _ := dsqle.PrepareCreateTableStmt(ctx, dsqle.NewUserSpaceDatabase(w.root, w.editOpts)) - createTableStmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, w.tableName) + createTableStmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, doltdb.TableName{Name: w.tableName}) if err != nil { return err } diff --git a/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go b/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go index b4c621bf3f..fd79bef236 100644 --- a/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go +++ b/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go @@ -129,7 +129,7 @@ func (w *SqlExportWriter) maybeWriteDropCreate(ctx context.Context) error { b.WriteString(sqlfmt.DropTableIfExistsStmt(w.tableName)) b.WriteRune('\n') sqlCtx, engine, _ := dsqle.PrepareCreateTableStmt(ctx, dsqle.NewUserSpaceDatabase(w.root, w.editOpts)) - createTableStmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, w.tableName) + createTableStmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, doltdb.TableName{Name: w.tableName}) if err != nil { return err } From 74c1208920ac08aa2cfd577ee9ec6957ef2e8444 Mon Sep 17 00:00:00 2001 From: Taylor Bantle Date: Tue, 24 Sep 2024 14:12:51 -0700 Subject: [PATCH 12/14] Fix dolt_commit_diff_$ --- go/libraries/doltcore/sqle/dtables/commit_diff_table.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/dtables/commit_diff_table.go b/go/libraries/doltcore/sqle/dtables/commit_diff_table.go index 7406ad64fa..886394a814 100644 --- a/go/libraries/doltcore/sqle/dtables/commit_diff_table.go +++ b/go/libraries/doltcore/sqle/dtables/commit_diff_table.go @@ -27,6 +27,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/rowconv" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/store/types" ) @@ -60,12 +61,11 @@ var _ sql.StatisticsTable = (*CommitDiffTable)(nil) func NewCommitDiffTable(ctx *sql.Context, dbName, tblName string, ddb *doltdb.DoltDB, wRoot, sRoot doltdb.RootValue) (sql.Table, error) { diffTblName := doltdb.DoltCommitDiffTablePrefix + tblName - // TODO: schema - table, _, ok, err := doltdb.GetTableInsensitive(ctx, wRoot, doltdb.TableName{Name: tblName}) + _, table, tableExists, err := resolve.Table(ctx, wRoot, tblName) if err != nil { return nil, err } - if !ok { + if !tableExists { return nil, sql.ErrTableNotFound.New(diffTblName) } From 27a3056cae2ce98c4c9318b9270117a0f8122eb8 Mon Sep 17 00:00:00 2001 From: Taylor Bantle Date: Tue, 24 Sep 2024 14:49:37 -0700 Subject: [PATCH 13/14] Revert GetCreateTableStmt changes --- go/cmd/dolt/commands/schcmds/export.go | 2 +- go/cmd/dolt/commands/schcmds/import.go | 2 +- go/cmd/dolt/commands/schcmds/show.go | 2 +- go/libraries/doltcore/sqle/show_create_table.go | 5 ++--- .../doltcore/table/untyped/sqlexport/batch_sqlwriter.go | 2 +- go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go | 2 +- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/go/cmd/dolt/commands/schcmds/export.go b/go/cmd/dolt/commands/schcmds/export.go index f5af2f6c8c..300f1deb4e 100644 --- a/go/cmd/dolt/commands/schcmds/export.go +++ b/go/cmd/dolt/commands/schcmds/export.go @@ -151,7 +151,7 @@ func exportSchemas(ctx context.Context, apr *argparser.ArgParseResults, root dol func exportTblSchema(ctx context.Context, tblName string, root doltdb.RootValue, wr io.Writer, opts editor.Options) errhand.VerboseError { sqlCtx, engine, _ := dsqle.PrepareCreateTableStmt(ctx, dsqle.NewUserSpaceDatabase(root, opts)) - stmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, doltdb.TableName{Name: tblName}) + stmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, tblName) if err != nil { return errhand.VerboseErrorFromError(err) } diff --git a/go/cmd/dolt/commands/schcmds/import.go b/go/cmd/dolt/commands/schcmds/import.go index 43be2127ca..3937bc41d7 100644 --- a/go/cmd/dolt/commands/schcmds/import.go +++ b/go/cmd/dolt/commands/schcmds/import.go @@ -306,7 +306,7 @@ func importSchema(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPars sqlDb := sqle.NewUserSpaceDatabase(root, editor.Options{}) sqlCtx, engine, _ := sqle.PrepareCreateTableStmt(ctx, sqlDb) - stmt, err := sqle.GetCreateTableStmt(sqlCtx, engine, doltdb.TableName{Name: tblName}) + stmt, err := sqle.GetCreateTableStmt(sqlCtx, engine, tblName) if err != nil { return errhand.VerboseErrorFromError(err) } diff --git a/go/cmd/dolt/commands/schcmds/show.go b/go/cmd/dolt/commands/schcmds/show.go index b56d1da024..4e8824123f 100644 --- a/go/cmd/dolt/commands/schcmds/show.go +++ b/go/cmd/dolt/commands/schcmds/show.go @@ -154,7 +154,7 @@ func printSchemas(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env notFound = append(notFound, tblName) } else { cli.Println(bold.Sprint(tblName), "@", cmStr) - stmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, doltdb.TableName{Name: tblName}) + stmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, tblName) if err != nil { return errhand.VerboseErrorFromError(err) } diff --git a/go/libraries/doltcore/sqle/show_create_table.go b/go/libraries/doltcore/sqle/show_create_table.go index 0d6d0103be..deab24085c 100644 --- a/go/libraries/doltcore/sqle/show_create_table.go +++ b/go/libraries/doltcore/sqle/show_create_table.go @@ -21,7 +21,6 @@ import ( sqle "github.com/dolthub/go-mysql-server" "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer" @@ -42,8 +41,8 @@ func PrepareCreateTableStmt(ctx context.Context, sqlDb dsess.SqlDatabase) (*sql. return sqlCtx, engine, sess } -func GetCreateTableStmt(ctx *sql.Context, engine *sqle.Engine, tableName doltdb.TableName) (string, error) { - _, rowIter, _, err := engine.Query(ctx, fmt.Sprintf("SHOW CREATE TABLE `%s`;", tableName.String())) +func GetCreateTableStmt(ctx *sql.Context, engine *sqle.Engine, tableName string) (string, error) { + _, rowIter, _, err := engine.Query(ctx, fmt.Sprintf("SHOW CREATE TABLE `%s`;", tableName)) if err != nil { return "", err } diff --git a/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go b/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go index f4c4617c4f..32aca3c252 100644 --- a/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go +++ b/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go @@ -154,7 +154,7 @@ func (w *BatchSqlExportWriter) maybeWriteDropCreate(ctx context.Context) error { b.WriteString(sqlfmt.DropTableIfExistsStmt(w.tableName)) b.WriteRune('\n') sqlCtx, engine, _ := dsqle.PrepareCreateTableStmt(ctx, dsqle.NewUserSpaceDatabase(w.root, w.editOpts)) - createTableStmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, doltdb.TableName{Name: w.tableName}) + createTableStmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, w.tableName) if err != nil { return err } diff --git a/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go b/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go index fd79bef236..b4c621bf3f 100644 --- a/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go +++ b/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go @@ -129,7 +129,7 @@ func (w *SqlExportWriter) maybeWriteDropCreate(ctx context.Context) error { b.WriteString(sqlfmt.DropTableIfExistsStmt(w.tableName)) b.WriteRune('\n') sqlCtx, engine, _ := dsqle.PrepareCreateTableStmt(ctx, dsqle.NewUserSpaceDatabase(w.root, w.editOpts)) - createTableStmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, doltdb.TableName{Name: w.tableName}) + createTableStmt, err := dsqle.GetCreateTableStmt(sqlCtx, engine, w.tableName) if err != nil { return err } From bd9581656bc26e98f6e7d37bc0e17eabeef81170 Mon Sep 17 00:00:00 2001 From: Maximilian Hoffman Date: Tue, 24 Sep 2024 15:12:16 -0700 Subject: [PATCH 14/14] Stats safer encode (#8379) * [no-release-notes] tests for varbinary stats * bump * bump * mcv row types fix * fix bats * nick's comments * [statsnoms] safer encode * fix loadBoundRow * revert testing edits * bad merge * [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh * try to fix race * revert testing change --------- Co-authored-by: max-hoffman --- .../doltcore/sqle/enginetest/stats_queries.go | 19 +++++++ go/libraries/doltcore/sqle/statsnoms/load.go | 52 +++++++++---------- go/libraries/doltcore/sqle/statsnoms/write.go | 32 +++++++++++- .../doltcore/sqle/statspro/dolt_stats.go | 2 + go/libraries/doltcore/sqle/statspro/update.go | 3 ++ integration-tests/bats/stats.bats | 14 +++++ 6 files changed, 93 insertions(+), 29 deletions(-) diff --git a/go/libraries/doltcore/sqle/enginetest/stats_queries.go b/go/libraries/doltcore/sqle/enginetest/stats_queries.go index 74825c251f..efe147b611 100644 --- a/go/libraries/doltcore/sqle/enginetest/stats_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/stats_queries.go @@ -331,6 +331,22 @@ var DoltStatsIOTests = []queries.ScriptTest{ }, }, }, + { + Name: "varbinary encoding bug", + SetUpScript: []string{ + `create table a (a varbinary (32) primary key)`, + "insert into a values ('hello, world')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: `analyze table a`, + }, + { + Query: "select count(*) from dolt_statistics", + Expected: []sql.Row{{1}}, + }, + }, + }, { Name: "boundary nils don't panic when trying to convert to the zero type", SetUpScript: []string{ @@ -590,6 +606,9 @@ var StatBranchTests = []queries.ScriptTest{ { Query: "call dolt_stats_stop()", }, + { + Query: "select sleep(.1)", + }, { Query: "call dolt_stats_drop()", }, diff --git a/go/libraries/doltcore/sqle/statsnoms/load.go b/go/libraries/doltcore/sqle/statsnoms/load.go index 536e88efa0..a85e05fa40 100644 --- a/go/libraries/doltcore/sqle/statsnoms/load.go +++ b/go/libraries/doltcore/sqle/statsnoms/load.go @@ -39,13 +39,11 @@ import ( func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.StatQualifier]*statspro.DoltStats, error) { qualToStats := make(map[sql.StatQualifier]*statspro.DoltStats) - iter, err := NewStatsIter(ctx, m) if err != nil { return nil, err } currentStat := statspro.NewDoltStats() - var lowerBound sql.Row for { row, err := iter.Next(ctx) if errors.Is(err, io.EOF) { @@ -108,15 +106,10 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St } } - boundRow, err := iter.ParseRow(boundRowStr) - if err != nil { - return nil, err - } - qual := sql.NewStatQualifier(dbName, tableName, indexName) if currentStat.Statistic.Qual.String() != qual.String() { if !currentStat.Statistic.Qual.Empty() { - currentStat.Statistic.LowerBnd, err = loadLowerBound(ctx, currentStat.Statistic.Qual) + currentStat.Statistic.LowerBnd, currentStat.Tb, err = loadLowerBound(ctx, db, currentStat.Statistic.Qual, len(currentStat.Columns())) if err != nil { return nil, err } @@ -133,7 +126,10 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St currentStat = statspro.NewDoltStats() currentStat.Statistic.Qual = qual currentStat.Statistic.Cols = columns - currentStat.Statistic.LowerBnd = lowerBound + currentStat.Statistic.LowerBnd, currentStat.Tb, err = loadLowerBound(ctx, db, currentStat.Statistic.Qual, len(currentStat.Columns())) + if err != nil { + return nil, err + } } if currentStat.Statistic.Hist == nil { @@ -144,6 +140,11 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St currentStat.Statistic.Qual = qual } + boundRow, err := DecodeRow(ctx, m.NodeStore(), boundRowStr, currentStat.Tb) + if err != nil { + return nil, err + } + bucket := statspro.DoltBucket{ Chunk: commit, Created: createdAt, @@ -166,7 +167,7 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St currentStat.Statistic.Created = createdAt } } - currentStat.Statistic.LowerBnd, err = loadLowerBound(ctx, currentStat.Statistic.Qual) + currentStat.Statistic.LowerBnd, currentStat.Tb, err = loadLowerBound(ctx, db, currentStat.Statistic.Qual, len(currentStat.Columns())) if err != nil { return nil, err } @@ -193,19 +194,14 @@ func parseTypeStrings(typs []string) ([]sql.Type, error) { return ret, nil } -func loadLowerBound(ctx *sql.Context, qual sql.StatQualifier) (sql.Row, error) { - dSess := dsess.DSessFromSess(ctx.Session) - roots, ok := dSess.GetRoots(ctx, qual.Db()) +func loadLowerBound(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifier, cols int) (sql.Row, *val.TupleBuilder, error) { + root, err := db.GetRoot(ctx) + table, ok, err := root.GetTable(ctx, doltdb.TableName{Name: qual.Table()}) if !ok { - return nil, nil - } - - table, ok, err := roots.Head.GetTable(ctx, doltdb.TableName{Name: qual.Table()}) - if !ok { - return nil, nil + return nil, nil, sql.ErrTableNotFound.New(qual.Table()) } if err != nil { - return nil, err + return nil, nil, err } var idx durable.Index @@ -215,25 +211,25 @@ func loadLowerBound(ctx *sql.Context, qual sql.StatQualifier) (sql.Row, error) { idx, err = table.GetIndexRowData(ctx, qual.Index()) } if err != nil { - return nil, err + return nil, nil, err } prollyMap := durable.ProllyMapFromIndex(idx) - keyBuilder := val.NewTupleBuilder(prollyMap.KeyDesc()) + keyBuilder := val.NewTupleBuilder(prollyMap.KeyDesc().PrefixDesc(cols)) buffPool := prollyMap.NodeStore().Pool() if cnt, err := prollyMap.Count(); err != nil { - return nil, err + return nil, nil, err } else if cnt == 0 { - return nil, nil + return nil, keyBuilder, nil } firstIter, err := prollyMap.IterOrdinalRange(ctx, 0, 1) if err != nil { - return nil, err + return nil, nil, err } keyBytes, _, err := firstIter.Next(ctx) if err != nil { - return nil, err + return nil, nil, err } for i := range keyBuilder.Desc.Types { keyBuilder.PutRaw(i, keyBytes.GetField(i)) @@ -244,10 +240,10 @@ func loadLowerBound(ctx *sql.Context, qual sql.StatQualifier) (sql.Row, error) { for i := 0; i < keyBuilder.Desc.Count(); i++ { firstRow[i], err = tree.GetField(ctx, prollyMap.KeyDesc(), i, firstKey, prollyMap.NodeStore()) if err != nil { - return nil, err + return nil, nil, err } } - return firstRow, nil + return firstRow, keyBuilder, nil } func loadFuncDeps(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifier) (*sql.FuncDepSet, sql.ColSet, error) { diff --git a/go/libraries/doltcore/sqle/statsnoms/write.go b/go/libraries/doltcore/sqle/statsnoms/write.go index bb48ee2244..b549dcbd59 100644 --- a/go/libraries/doltcore/sqle/statsnoms/write.go +++ b/go/libraries/doltcore/sqle/statsnoms/write.go @@ -27,6 +27,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/statspro" "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/val" ) @@ -120,7 +121,11 @@ func putIndexRows(ctx context.Context, statsMap *prolly.MutableMap, dStats *stat valueBuilder.PutInt64(4, int64(h.NullCount())) valueBuilder.PutString(5, strings.Join(dStats.Columns(), ",")) valueBuilder.PutString(6, typesStr) - valueBuilder.PutString(7, stats.StringifyKey(h.UpperBound(), dStats.Statistic.Typs)) + boundRow, err := EncodeRow(ctx, statsMap.NodeStore(), h.UpperBound(), dStats.Tb) + if err != nil { + return err + } + valueBuilder.PutString(7, string(boundRow)) valueBuilder.PutInt64(8, int64(h.BoundCount())) valueBuilder.PutDatetime(9, statspro.DoltBucketCreated(h)) for i, r := range h.Mcvs() { @@ -139,3 +144,28 @@ func putIndexRows(ctx context.Context, statsMap *prolly.MutableMap, dStats *stat } return nil } + +func EncodeRow(ctx context.Context, ns tree.NodeStore, r sql.Row, tb *val.TupleBuilder) ([]byte, error) { + for i, v := range r { + if v == nil { + continue + } + if err := tree.PutField(ctx, ns, tb, i, v); err != nil { + return nil, err + } + } + return tb.Build(ns.Pool()), nil +} + +func DecodeRow(ctx context.Context, ns tree.NodeStore, s string, tb *val.TupleBuilder) (sql.Row, error) { + tup := []byte(s) + r := make(sql.Row, tb.Desc.Count()) + var err error + for i, _ := range r { + r[i], err = tree.GetField(ctx, tb.Desc, i, tup, ns) + if err != nil { + return nil, err + } + } + return r, nil +} diff --git a/go/libraries/doltcore/sqle/statspro/dolt_stats.go b/go/libraries/doltcore/sqle/statspro/dolt_stats.go index eb1107873c..4c5d43250c 100644 --- a/go/libraries/doltcore/sqle/statspro/dolt_stats.go +++ b/go/libraries/doltcore/sqle/statspro/dolt_stats.go @@ -24,6 +24,7 @@ import ( "github.com/dolthub/go-mysql-server/sql/stats" "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/val" ) type DoltStats struct { @@ -36,6 +37,7 @@ type DoltStats struct { // field on disk Active map[hash.Hash]int Hist sql.Histogram + Tb *val.TupleBuilder } func (s *DoltStats) Clone(_ context.Context) sql.JSONWrapper { diff --git a/go/libraries/doltcore/sqle/statspro/update.go b/go/libraries/doltcore/sqle/statspro/update.go index 6da109bb68..562e82c567 100644 --- a/go/libraries/doltcore/sqle/statspro/update.go +++ b/go/libraries/doltcore/sqle/statspro/update.go @@ -90,6 +90,8 @@ func createNewStatsBuckets(ctx *sql.Context, sqlTable sql.Table, dTab *doltdb.Ta ret[meta.qual].Statistic.Fds = fds ret[meta.qual].Statistic.Colset = colSet + ret[meta.qual].Tb = val.NewTupleBuilder(prollyMap.KeyDesc().PrefixDesc(len(meta.cols))) + continue } @@ -105,6 +107,7 @@ func createNewStatsBuckets(ctx *sql.Context, sqlTable sql.Table, dTab *doltdb.Ta ret[meta.qual].Statistic.Cols = meta.cols ret[meta.qual].Statistic.Typs = types ret[meta.qual].Statistic.Qual = meta.qual + ret[meta.qual].Tb = val.NewTupleBuilder(prollyMap.KeyDesc().PrefixDesc(len(meta.cols))) var start, stop uint64 // read leaf rows for each bucket diff --git a/integration-tests/bats/stats.bats b/integration-tests/bats/stats.bats index 973d23a794..0d1149d584 100644 --- a/integration-tests/bats/stats.bats +++ b/integration-tests/bats/stats.bats @@ -116,6 +116,20 @@ teardown() { [ "${lines[1]}" = "0" ] } +@test "stats: encode/decode loop is delimiter safe" { + cd repo2 + +dolt sql <