mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2025-12-30 16:12:39 -06:00
Code Issues Packages Projects Releases 114 Wiki Activity

Merge branch 'main' into zachmu/reset-2

Browse Source
This commit is contained in:
Zach Musgrave
2024-09-25 11:40:49 -07:00
parent 7b771cfbef a63bf91ca4
commit bb6cf2a50d
18 changed files with 178 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
go/go.mod
View File

@@ -15,7 +15,7 @@ require (
github.com/dolthub/fslock v0.0.3
github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
github.com/dolthub/vitess v0.0.0-20240916204416-9d4d4a09b1d9
github.com/dolthub/vitess v0.0.0-20240919225659-2ad81685e772
github.com/dustin/go-humanize v1.0.1
github.com/fatih/color v1.13.0
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
@@ -57,7 +57,7 @@ require (
github.com/cespare/xxhash/v2 v2.2.0
github.com/creasty/defaults v1.6.0
github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
github.com/dolthub/go-mysql-server v0.18.2-0.20240919175427-5055c7114c8b
github.com/dolthub/go-mysql-server v0.18.2-0.20240923181307-5aacdb13e45a
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63
github.com/dolthub/swiss v0.1.0
github.com/goccy/go-json v0.10.2

8
go/go.sum
View File

@@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662 h1:aC17hZD6iwzBwwfO5M+3oBT5E5gGRiQPdn+vzpDXqIA=
github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
github.com/dolthub/go-mysql-server v0.18.2-0.20240919175427-5055c7114c8b h1:y5oKakoZEsTZCDu67VOD395uhKG00VpVGzSYTZoYiDY=
github.com/dolthub/go-mysql-server v0.18.2-0.20240919175427-5055c7114c8b/go.mod h1:m88EMm9OthVVa6qIhbpnRDpj/eYUXuNpvY/+0YWKVwc=
github.com/dolthub/go-mysql-server v0.18.2-0.20240923181307-5aacdb13e45a h1:rpCmZj332eiBbzsHsq3Sj5AWzl3Q7szDObwI49UqA8Y=
github.com/dolthub/go-mysql-server v0.18.2-0.20240923181307-5aacdb13e45a/go.mod h1:lGbU2bK+QNnlETdUjOOaE+UnlEUu31VaQOFKAFGyZN4=
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=
@@ -197,8 +197,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY=
github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
github.com/dolthub/vitess v0.0.0-20240916204416-9d4d4a09b1d9 h1:2My8cED5m5/sFay7U4bvLxpECJccKj0cEKCqEA+63yU=
github.com/dolthub/vitess v0.0.0-20240916204416-9d4d4a09b1d9/go.mod h1:uBvlRluuL+SbEWTCZ68o0xvsdYZER3CEG/35INdzfJM=
github.com/dolthub/vitess v0.0.0-20240919225659-2ad81685e772 h1:vDwBX7Lc8DnA8Zk0iRIu6slCw0GIUfYfFlYDYJQw8GQ=
github.com/dolthub/vitess v0.0.0-20240919225659-2ad81685e772/go.mod h1:uBvlRluuL+SbEWTCZ68o0xvsdYZER3CEG/35INdzfJM=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=

31
go/libraries/doltcore/env/actions/checkout.go vendored
View File

@@ -24,6 +24,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
"github.com/dolthub/dolt/go/libraries/utils/set"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
@@ -88,6 +89,36 @@ func MoveTablesFromHeadToWorking(ctx context.Context, roots doltdb.Roots, tbls [
return roots, nil
}
// FindTableInRoots resolves a table by looking in all three roots (working,
// staged, head) in that order.
func FindTableInRoots(ctx *sql.Context, roots doltdb.Roots, name string) (doltdb.TableName, *doltdb.Table, bool, error) {
tbl, root, tblExists, err := resolve.Table(ctx, roots.Working, name)
if err != nil {
return doltdb.TableName{}, nil, false, err
}
if tblExists {
return tbl, root, true, nil
}
tbl, root, tblExists, err = resolve.Table(ctx, roots.Staged, name)
if err != nil {
return doltdb.TableName{}, nil, false, err
}
if tblExists {
return tbl, root, true, nil
}
tbl, root, tblExists, err = resolve.Table(ctx, roots.Head, name)
if err != nil {
return doltdb.TableName{}, nil, false, err
}
if tblExists {
return tbl, root, true, nil
}
return doltdb.TableName{}, nil, false, nil
}
// RootsForBranch returns the roots needed for a branch checkout. |roots.Head| should be the pre-checkout head. The
// returned roots struct has |Head| set to |branchRoot|.
func RootsForBranch(ctx context.Context, roots doltdb.Roots, branchRoot doltdb.RootValue, force bool) (doltdb.Roots, error) {

14
go/libraries/doltcore/env/actions/reset.go vendored
View File

@@ -25,6 +25,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
"github.com/dolthub/dolt/go/store/datas"
)
@@ -271,7 +272,7 @@ func getUnionedTables(ctx context.Context, tables []doltdb.TableName, stagedRoot
// CleanUntracked deletes untracked tables from the working root.
// Evaluates untracked tables as: all working tables - all staged tables.
func CleanUntracked(ctx context.Context, roots doltdb.Roots, tables []string, dryrun bool, force bool) (doltdb.Roots, error) {
func CleanUntracked(ctx *sql.Context, roots doltdb.Roots, tables []string, dryrun bool, force bool) (doltdb.Roots, error) {
untrackedTables := make(map[doltdb.TableName]struct{})
var err error
@@ -284,21 +285,24 @@ func CleanUntracked(ctx context.Context, roots doltdb.Roots, tables []string, dr
for i := range tables {
name := tables[i]
_, _, err = roots.Working.GetTable(ctx, doltdb.TableName{Name: name})
resolvedName, _, tblExists, err := resolve.Table(ctx, roots.Working, name)
if err != nil {
return doltdb.Roots{}, err
}
untrackedTables[doltdb.TableName{Name: name}] = struct{}{}
if !tblExists {
return doltdb.Roots{}, fmt.Errorf("%w: '%s'", doltdb.ErrTableNotFound, name)
}
untrackedTables[resolvedName] = struct{}{}
}
// untracked tables = working tables - staged tables
headTblNames, err := roots.Staged.GetTableNames(ctx, doltdb.DefaultSchemaName)
headTblNames := GetAllTableNames(ctx, roots.Staged)
if err != nil {
return doltdb.Roots{}, err
}
for _, name := range headTblNames {
delete(untrackedTables, doltdb.TableName{Name: name})
delete(untrackedTables, name)
}
newRoot := roots.Working

7
go/libraries/doltcore/remotestorage/internal/reliable/http.go
View File

@@ -221,8 +221,11 @@ func StreamingRangeDownload(ctx context.Context, req StreamingRangeRequest) Stre
}()
return StreamingResponse{
Body: r,
cancel: cancel,
Body: r,
cancel: func() {
r.Close()
cancel()
},
}
}

17
go/libraries/doltcore/sqle/dolt_diff_stat_table_function.go
View File

@@ -293,7 +293,11 @@ func (ds *DiffStatTableFunction) RowIter(ctx *sql.Context, row sql.Row) (sql.Row
// If tableNameExpr defined, return a single table diff stat result
if ds.tableNameExpr != nil {
delta := findMatchingDelta(deltas, tableName)
diffStat, hasDiff, err := getDiffStatNodeFromDelta(ctx, delta, fromRefDetails.root, toRefDetails.root, tableName)
schemaName := delta.FromName.Schema
if schemaName == "" {
schemaName = delta.ToName.Schema
}
diffStat, hasDiff, err := getDiffStatNodeFromDelta(ctx, delta, fromRefDetails.root, toRefDetails.root, doltdb.TableName{Name: tableName, Schema: schemaName})
if err != nil {
return nil, err
}
@@ -309,8 +313,7 @@ func (ds *DiffStatTableFunction) RowIter(ctx *sql.Context, row sql.Row) (sql.Row
if tblName.Name == "" {
tblName = delta.FromName
}
// TODO: schema name
diffStat, hasDiff, err := getDiffStatNodeFromDelta(ctx, delta, fromRefDetails.root, toRefDetails.root, tblName.Name)
diffStat, hasDiff, err := getDiffStatNodeFromDelta(ctx, delta, fromRefDetails.root, toRefDetails.root, tblName)
if err != nil {
if errors.Is(err, diff.ErrPrimaryKeySetChanged) {
ctx.Warn(dtables.PrimaryKeyChangeWarningCode, fmt.Sprintf("stat for table %s cannot be determined. Primary key set changed.", tblName))
@@ -369,10 +372,10 @@ func (ds *DiffStatTableFunction) evaluateArguments() (interface{}, interface{},
// getDiffStatNodeFromDelta returns diffStatNode object and whether there is data diff or not. It gets tables
// from roots and diff stat if there is a valid table exists in both fromRoot and toRoot.
func getDiffStatNodeFromDelta(ctx *sql.Context, delta diff.TableDelta, fromRoot, toRoot doltdb.RootValue, tableName string) (diffStatNode, bool, error) {
func getDiffStatNodeFromDelta(ctx *sql.Context, delta diff.TableDelta, fromRoot, toRoot doltdb.RootValue, tableName doltdb.TableName) (diffStatNode, bool, error) {
var oldColLen int
var newColLen int
fromTable, _, fromTableExists, err := doltdb.GetTableInsensitive(ctx, fromRoot, doltdb.TableName{Name: tableName})
fromTable, _, fromTableExists, err := doltdb.GetTableInsensitive(ctx, fromRoot, tableName)
if err != nil {
return diffStatNode{}, false, err
}
@@ -385,7 +388,7 @@ func getDiffStatNodeFromDelta(ctx *sql.Context, delta diff.TableDelta, fromRoot,
oldColLen = len(fromSch.GetAllCols().GetColumns())
}
toTable, _, toTableExists, err := doltdb.GetTableInsensitive(ctx, toRoot, doltdb.TableName{Name: tableName})
toTable, _, toTableExists, err := doltdb.GetTableInsensitive(ctx, toRoot, tableName)
if err != nil {
return diffStatNode{}, false, err
}
@@ -412,7 +415,7 @@ func getDiffStatNodeFromDelta(ctx *sql.Context, delta diff.TableDelta, fromRoot,
return diffStatNode{}, false, err
}
return diffStatNode{tableName, diffStat, oldColLen, newColLen, keyless}, hasDiff, nil
return diffStatNode{tableName.Name, diffStat, oldColLen, newColLen, keyless}, hasDiff, nil
}
// getDiffStat returns diff.DiffStatProgress object and whether there is a data diff or not.

6
go/libraries/doltcore/sqle/dolt_patch_table_function.go
View File

@@ -29,12 +29,12 @@ import (
"golang.org/x/exp/slices"
"github.com/dolthub/dolt/go/libraries/doltcore/diff"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlfmt"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
"github.com/dolthub/dolt/go/store/types"
@@ -145,11 +145,11 @@ func (p *PatchTableFunction) PartitionRows(ctx *sql.Context, partition sql.Parti
// If tableNameExpr defined, return a single table patch result
if p.tableNameExpr != nil {
fromTblExists, err := fromRefDetails.root.HasTable(ctx, doltdb.TableName{Name: tableName})
_, _, fromTblExists, err := resolve.Table(ctx, fromRefDetails.root, tableName)
if err != nil {
return nil, err
}
toTblExists, err := toRefDetails.root.HasTable(ctx, doltdb.TableName{Name: tableName})
_, _, toTblExists, err := resolve.Table(ctx, toRefDetails.root, tableName)
if err != nil {
return nil, err
}

10
go/libraries/doltcore/sqle/dolt_schema_diff_table_function.go
View File

@@ -27,7 +27,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/diff"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlfmt"
)
const schemaDiffDefaultRowCount = 100
@@ -314,18 +314,14 @@ func (ds *SchemaDiffTableFunction) RowIter(ctx *sql.Context, row sql.Row) (sql.R
var fromCreate, toCreate string
if delta.FromTable != nil {
fromSqlDb := NewUserSpaceDatabase(fromRoot, editor.Options{})
fromSqlCtx, fromEngine, _ := PrepareCreateTableStmt(ctx, fromSqlDb)
fromCreate, err = GetCreateTableStmt(fromSqlCtx, fromEngine, delta.FromName.Name)
fromCreate, err = sqlfmt.GenerateCreateTableStatement(delta.FromName.Name, delta.FromSch, delta.FromFks, delta.FromFksParentSch)
if err != nil {
return nil, err
}
}
if delta.ToTable != nil {
toSqlDb := NewUserSpaceDatabase(toRoot, editor.Options{})
toSqlCtx, toEngine, _ := PrepareCreateTableStmt(ctx, toSqlDb)
toCreate, err = GetCreateTableStmt(toSqlCtx, toEngine, delta.ToName.Name)
toCreate, err = sqlfmt.GenerateCreateTableStatement(delta.ToName.Name, delta.ToSch, delta.ToFks, delta.ToFksParentSch)
if err != nil {
return nil, err
}

15
go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go
View File

@@ -476,9 +476,20 @@ func doGlobalCheckout(ctx *sql.Context, branchName string, isForce bool, isNewBr
}
func checkoutTables(ctx *sql.Context, roots doltdb.Roots, name string, tables []string) error {
// TODO: schema name
roots, err := actions.MoveTablesFromHeadToWorking(ctx, roots, doltdb.ToTableNames(tables, doltdb.DefaultSchemaName))
tableNames := make([]doltdb.TableName, len(tables))
for i, table := range tables {
tbl, _, exists, err := actions.FindTableInRoots(ctx, roots, table)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("error: given tables do not exist")
}
tableNames[i] = tbl
}
roots, err := actions.MoveTablesFromHeadToWorking(ctx, roots, tableNames)
if err != nil {
if doltdb.IsRootValUnreachable(err) {
rt := doltdb.GetUnreachableRootType(err)

2
go/libraries/doltcore/sqle/dprocedures/dolt_reset.go
View File

@@ -17,7 +17,6 @@ package dprocedures
import (
"fmt"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/dolt/go/cmd/dolt/cli"
@@ -26,6 +25,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
"github.com/dolthub/dolt/go/libraries/utils/argparser"
)

6
go/libraries/doltcore/sqle/dtables/commit_diff_table.go
View File

@@ -27,6 +27,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/rowconv"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
"github.com/dolthub/dolt/go/store/types"
)
@@ -60,12 +61,11 @@ var _ sql.StatisticsTable = (*CommitDiffTable)(nil)
func NewCommitDiffTable(ctx *sql.Context, dbName, tblName string, ddb *doltdb.DoltDB, wRoot, sRoot doltdb.RootValue) (sql.Table, error) {
diffTblName := doltdb.DoltCommitDiffTablePrefix + tblName
// TODO: schema
table, _, ok, err := doltdb.GetTableInsensitive(ctx, wRoot, doltdb.TableName{Name: tblName})
_, table, tableExists, err := resolve.Table(ctx, wRoot, tblName)
if err != nil {
return nil, err
}
if !ok {
if !tableExists {
return nil, sql.ErrTableNotFound.New(diffTblName)
}

2
go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go
View File

@@ -17,7 +17,6 @@ package dtables
import (
"encoding/json"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
@@ -25,6 +24,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/merge"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/pool"

19
go/libraries/doltcore/sqle/enginetest/stats_queries.go
View File

@@ -331,6 +331,22 @@ var DoltStatsIOTests = []queries.ScriptTest{
},
},
},
{
Name: "varbinary encoding bug",
SetUpScript: []string{
`create table a (a varbinary (32) primary key)`,
"insert into a values ('hello, world')",
},
Assertions: []queries.ScriptTestAssertion{
{
Query: `analyze table a`,
},
{
Query: "select count(*) from dolt_statistics",
Expected: []sql.Row{{1}},
},
},
},
{
Name: "boundary nils don't panic when trying to convert to the zero type",
SetUpScript: []string{
@@ -590,6 +606,9 @@ var StatBranchTests = []queries.ScriptTest{
{
Query: "call dolt_stats_stop()",
},
{
Query: "select sleep(.1)",
},
{
Query: "call dolt_stats_drop()",
},

52
go/libraries/doltcore/sqle/statsnoms/load.go
View File

@@ -39,13 +39,11 @@ import (
func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.StatQualifier]*statspro.DoltStats, error) {
qualToStats := make(map[sql.StatQualifier]*statspro.DoltStats)
iter, err := NewStatsIter(ctx, m)
if err != nil {
return nil, err
}
currentStat := statspro.NewDoltStats()
var lowerBound sql.Row
for {
row, err := iter.Next(ctx)
if errors.Is(err, io.EOF) {
@@ -108,15 +106,10 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St
}
}
boundRow, err := iter.ParseRow(boundRowStr)
if err != nil {
return nil, err
}
qual := sql.NewStatQualifier(dbName, tableName, indexName)
if currentStat.Statistic.Qual.String() != qual.String() {
if !currentStat.Statistic.Qual.Empty() {
currentStat.Statistic.LowerBnd, err = loadLowerBound(ctx, currentStat.Statistic.Qual)
currentStat.Statistic.LowerBnd, currentStat.Tb, err = loadLowerBound(ctx, db, currentStat.Statistic.Qual, len(currentStat.Columns()))
if err != nil {
return nil, err
}
@@ -133,7 +126,10 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St
currentStat = statspro.NewDoltStats()
currentStat.Statistic.Qual = qual
currentStat.Statistic.Cols = columns
currentStat.Statistic.LowerBnd = lowerBound
currentStat.Statistic.LowerBnd, currentStat.Tb, err = loadLowerBound(ctx, db, currentStat.Statistic.Qual, len(currentStat.Columns()))
if err != nil {
return nil, err
}
}
if currentStat.Statistic.Hist == nil {
@@ -144,6 +140,11 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St
currentStat.Statistic.Qual = qual
}
boundRow, err := DecodeRow(ctx, m.NodeStore(), boundRowStr, currentStat.Tb)
if err != nil {
return nil, err
}
bucket := statspro.DoltBucket{
Chunk: commit,
Created: createdAt,
@@ -166,7 +167,7 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St
currentStat.Statistic.Created = createdAt
}
}
currentStat.Statistic.LowerBnd, err = loadLowerBound(ctx, currentStat.Statistic.Qual)
currentStat.Statistic.LowerBnd, currentStat.Tb, err = loadLowerBound(ctx, db, currentStat.Statistic.Qual, len(currentStat.Columns()))
if err != nil {
return nil, err
}
@@ -193,19 +194,14 @@ func parseTypeStrings(typs []string) ([]sql.Type, error) {
return ret, nil
}
func loadLowerBound(ctx *sql.Context, qual sql.StatQualifier) (sql.Row, error) {
dSess := dsess.DSessFromSess(ctx.Session)
roots, ok := dSess.GetRoots(ctx, qual.Db())
func loadLowerBound(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifier, cols int) (sql.Row, *val.TupleBuilder, error) {
root, err := db.GetRoot(ctx)
table, ok, err := root.GetTable(ctx, doltdb.TableName{Name: qual.Table()})
if !ok {
return nil, nil
}
table, ok, err := roots.Head.GetTable(ctx, doltdb.TableName{Name: qual.Table()})
if !ok {
return nil, nil
return nil, nil, sql.ErrTableNotFound.New(qual.Table())
}
if err != nil {
return nil, err
return nil, nil, err
}
var idx durable.Index
@@ -215,25 +211,25 @@ func loadLowerBound(ctx *sql.Context, qual sql.StatQualifier) (sql.Row, error) {
idx, err = table.GetIndexRowData(ctx, qual.Index())
}
if err != nil {
return nil, err
return nil, nil, err
}
prollyMap := durable.ProllyMapFromIndex(idx)
keyBuilder := val.NewTupleBuilder(prollyMap.KeyDesc())
keyBuilder := val.NewTupleBuilder(prollyMap.KeyDesc().PrefixDesc(cols))
buffPool := prollyMap.NodeStore().Pool()
if cnt, err := prollyMap.Count(); err != nil {
return nil, err
return nil, nil, err
} else if cnt == 0 {
return nil, nil
return nil, keyBuilder, nil
}
firstIter, err := prollyMap.IterOrdinalRange(ctx, 0, 1)
if err != nil {
return nil, err
return nil, nil, err
}
keyBytes, _, err := firstIter.Next(ctx)
if err != nil {
return nil, err
return nil, nil, err
}
for i := range keyBuilder.Desc.Types {
keyBuilder.PutRaw(i, keyBytes.GetField(i))
@@ -244,10 +240,10 @@ func loadLowerBound(ctx *sql.Context, qual sql.StatQualifier) (sql.Row, error) {
for i := 0; i < keyBuilder.Desc.Count(); i++ {
firstRow[i], err = tree.GetField(ctx, prollyMap.KeyDesc(), i, firstKey, prollyMap.NodeStore())
if err != nil {
return nil, err
return nil, nil, err
}
}
return firstRow, nil
return firstRow, keyBuilder, nil
}
func loadFuncDeps(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifier) (*sql.FuncDepSet, sql.ColSet, error) {

32
go/libraries/doltcore/sqle/statsnoms/write.go
View File

@@ -27,6 +27,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/statspro"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/val"
)
@@ -120,7 +121,11 @@ func putIndexRows(ctx context.Context, statsMap *prolly.MutableMap, dStats *stat
valueBuilder.PutInt64(4, int64(h.NullCount()))
valueBuilder.PutString(5, strings.Join(dStats.Columns(), ","))
valueBuilder.PutString(6, typesStr)
valueBuilder.PutString(7, stats.StringifyKey(h.UpperBound(), dStats.Statistic.Typs))
boundRow, err := EncodeRow(ctx, statsMap.NodeStore(), h.UpperBound(), dStats.Tb)
if err != nil {
return err
}
valueBuilder.PutString(7, string(boundRow))
valueBuilder.PutInt64(8, int64(h.BoundCount()))
valueBuilder.PutDatetime(9, statspro.DoltBucketCreated(h))
for i, r := range h.Mcvs() {
@@ -139,3 +144,28 @@ func putIndexRows(ctx context.Context, statsMap *prolly.MutableMap, dStats *stat
}
return nil
}
func EncodeRow(ctx context.Context, ns tree.NodeStore, r sql.Row, tb *val.TupleBuilder) ([]byte, error) {
for i, v := range r {
if v == nil {
continue
}
if err := tree.PutField(ctx, ns, tb, i, v); err != nil {
return nil, err
}
}
return tb.Build(ns.Pool()), nil
}
func DecodeRow(ctx context.Context, ns tree.NodeStore, s string, tb *val.TupleBuilder) (sql.Row, error) {
tup := []byte(s)
r := make(sql.Row, tb.Desc.Count())
var err error
for i, _ := range r {
r[i], err = tree.GetField(ctx, tb.Desc, i, tup, ns)
if err != nil {
return nil, err
}
}
return r, nil
}

2
go/libraries/doltcore/sqle/statspro/dolt_stats.go
View File

@@ -24,6 +24,7 @@ import (
"github.com/dolthub/go-mysql-server/sql/stats"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/val"
)
type DoltStats struct {
@@ -36,6 +37,7 @@ type DoltStats struct {
// field on disk
Active map[hash.Hash]int
Hist sql.Histogram
Tb *val.TupleBuilder
}
func (s *DoltStats) Clone(_ context.Context) sql.JSONWrapper {

3
go/libraries/doltcore/sqle/statspro/update.go
View File

@@ -90,6 +90,8 @@ func createNewStatsBuckets(ctx *sql.Context, sqlTable sql.Table, dTab *doltdb.Ta
ret[meta.qual].Statistic.Fds = fds
ret[meta.qual].Statistic.Colset = colSet
ret[meta.qual].Tb = val.NewTupleBuilder(prollyMap.KeyDesc().PrefixDesc(len(meta.cols)))
continue
}
@@ -105,6 +107,7 @@ func createNewStatsBuckets(ctx *sql.Context, sqlTable sql.Table, dTab *doltdb.Ta
ret[meta.qual].Statistic.Cols = meta.cols
ret[meta.qual].Statistic.Typs = types
ret[meta.qual].Statistic.Qual = meta.qual
ret[meta.qual].Tb = val.NewTupleBuilder(prollyMap.KeyDesc().PrefixDesc(len(meta.cols)))
var start, stop uint64
// read leaf rows for each bucket

14
integration-tests/bats/stats.bats
View File

@@ -116,6 +116,20 @@ teardown() {
[ "${lines[1]}" = "0" ]
}
@test "stats: encode/decode loop is delimiter safe" {
cd repo2
dolt sql <<EOF
create table uv (u varbinary(255) primary key);
insert into uv values ('hello, world');
analyze table uv;
EOF
run dolt sql -r csv -q "select count(*) from dolt_statistics"
[ "$status" -eq 0 ]
[ "${lines[1]}" = "1" ]
}
@test "stats: correct stats directory location, issue#8324" {
cd repo2