From c2e9161b2087dc23b2e72c3115274cfe7f020cb0 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 7 Mar 2022 09:41:10 -0800 Subject: [PATCH 1/8] update new format dolt index to use filtered index interface --- go/libraries/doltcore/sqle/index/dolt_index.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index 66122d5167..599c0496bb 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -290,7 +290,12 @@ RangeLoop: } func (di doltIndex) HandledFilters(filters []sql.Expression) []sql.Expression { - return filters + if types.IsFormat_DOLT_1(di.vrw.Format()) { + // todo(andy): handle first column filters + return nil + } else { + return filters + } } // Database implement sql.Index @@ -425,14 +430,13 @@ func prollyRangeFromSqlRange(sqlRange sql.Range, tb *val.TupleBuilder) (rng prol start := prolly.RangeCut{Inclusive: true} startRow := sql.Row{} for _, sc := range lower { - if !isBindingCut(sc) { + if !sql.RangeCutIsBinding(sc) { start = prolly.RangeCut{Unbound: true, Inclusive: false} break } start.Inclusive = start.Inclusive && sc.TypeAsLowerBound() == sql.Closed startRow = append(startRow, sql.GetRangeCutKey(sc)) } - if !start.Unbound { startRow, err = normalizeRangeKey(sqlRange, startRow) if err != nil { @@ -448,7 +452,7 @@ func prollyRangeFromSqlRange(sqlRange sql.Range, tb *val.TupleBuilder) (rng prol stop := prolly.RangeCut{Inclusive: true} stopRow := sql.Row{} for _, sc := range upper { - if !isBindingCut(sc) { + if !sql.RangeCutIsBinding(sc) { stop = prolly.RangeCut{Unbound: true, Inclusive: false} break } @@ -476,10 +480,6 @@ func prollyRangeFromSqlRange(sqlRange sql.Range, tb *val.TupleBuilder) (rng prol }, nil } -func isBindingCut(cut sql.RangeCut) bool { - return cut != sql.BelowAll{} && cut != sql.AboveAll{} -} - func tupleFromKeys(keys sql.Row, tb *val.TupleBuilder) (val.Tuple, error) { var err error for i, v := range keys { From 064db81bdd5615f72914790a714fb6f3c397824a Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 7 Mar 2022 13:40:10 -0800 Subject: [PATCH 2/8] fix table truncation --- go/libraries/doltcore/sqle/tables.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index 54f544b04a..d933994559 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -484,6 +484,11 @@ func (t *WritableDoltTable) Truncate(ctx *sql.Context) (int, error) { return 0, err } + sch, err := table.GetSchema(ctx) + if err != nil { + return 0, err + } + rowData, err := table.GetRowData(ctx) if err != nil { return 0, err @@ -495,13 +500,20 @@ func (t *WritableDoltTable) Truncate(ctx *sql.Context) (int, error) { return 0, err } - // truncate table resets auto-increment value - newTable, err := doltdb.NewTable(ctx, table.ValueReadWriter(), t.sch, empty, nil, nil) + idxSet, err := table.GetIndexSet(ctx) if err != nil { return 0, err } - newTable, err = editor.RebuildAllIndexes(ctx, newTable, t.opts) + for _, idx := range sch.Indexes().AllIndexes() { + idxSet, err = idxSet.PutIndex(ctx, idx.Name(), empty) + if err != nil { + return 0, err + } + } + + // truncate table resets auto-increment value + newTable, err := doltdb.NewTable(ctx, table.ValueReadWriter(), t.sch, empty, idxSet, nil) if err != nil { return 0, err } From 792dbe3f71a516c23b46a76f079d4b1c54406a3c Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 7 Mar 2022 15:17:34 -0800 Subject: [PATCH 3/8] truncate tests working other than foreign key tests --- go/libraries/doltcore/sqle/database.go | 26 +----- .../doltcore/sqle/index/dolt_index_test.go | 8 +- .../doltcore/sqle/index/index_lookup.go | 29 ++++-- .../doltcore/sqle/index/prolly_index_iter.go | 2 +- .../doltcore/sqle/indexed_dolt_table.go | 2 +- .../doltcore/sqle/procedures_table.go | 2 +- go/libraries/doltcore/sqle/schema_table.go | 93 ++++++++++++++++--- go/store/prolly/map.go | 13 +++ go/store/prolly/node_cursor.go | 10 +- 9 files changed, 133 insertions(+), 52 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 39e3d21a61..be52a91da9 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -925,13 +925,10 @@ func (db Database) GetView(ctx *sql.Context, viewName string) (string, bool, err return view, true, nil } - tbl, ok, err := root.GetTable(ctx, doltdb.SchemasTableName) + tbl, err := GetOrCreateDoltSchemasTable(ctx, db) if err != nil { return "", false, err } - if !ok { - return "", false, nil - } fragments, err := getSchemaFragmentsOfType(ctx, tbl, "view") if err != nil { @@ -949,19 +946,11 @@ func (db Database) GetView(ctx *sql.Context, viewName string) (string, bool, err // GetView implements sql.ViewDatabase func (db Database) AllViews(ctx *sql.Context) ([]sql.ViewDefinition, error) { - root, err := db.GetRoot(ctx) + tbl, err := GetOrCreateDoltSchemasTable(ctx, db) if err != nil { return nil, err } - tbl, ok, err := root.GetTable(ctx, doltdb.SchemasTableName) - if err != nil { - return nil, err - } - if !ok { - return nil, nil - } - frags, err := getSchemaFragmentsOfType(ctx, tbl, "view") if err != nil { return nil, err @@ -997,19 +986,10 @@ func (db Database) DropView(ctx *sql.Context, name string) error { // GetTriggers implements sql.TriggerDatabase. func (db Database) GetTriggers(ctx *sql.Context) ([]sql.TriggerDefinition, error) { - root, err := db.GetRoot(ctx) + tbl, err := GetOrCreateDoltSchemasTable(ctx, db) if err != nil { return nil, err } - - tbl, ok, err := root.GetTable(ctx, doltdb.SchemasTableName) - if err != nil { - return nil, err - } - if !ok { - return nil, nil - } - frags, err := getSchemaFragmentsOfType(ctx, tbl, "trigger") if err != nil { return nil, err diff --git a/go/libraries/doltcore/sqle/index/dolt_index_test.go b/go/libraries/doltcore/sqle/index/dolt_index_test.go index 471ca46474..8f891bfb88 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index_test.go +++ b/go/libraries/doltcore/sqle/index/dolt_index_test.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "io" "sort" "strings" @@ -1056,7 +1057,10 @@ func TestDoltIndexBetween(t *testing.T) { indexLookup, err := sqlIndex.Build(ctx) require.NoError(t, err) - indexIter, err := index.RowIterForIndexLookup(ctx, indexLookup, nil) + pkSch, err := sqlutil.FromDoltSchema("fake_table", idx.Schema()) + require.NoError(t, err) + + indexIter, err := index.RowIterForIndexLookup(ctx, indexLookup, pkSch, nil) require.NoError(t, err) var readRows []sql.Row @@ -1275,7 +1279,7 @@ func testDoltIndex(t *testing.T, keys []interface{}, expectedRows []sql.Row, idx indexLookup, err := builder.Build(ctx) require.NoError(t, err) - indexIter, err := index.RowIterForIndexLookup(ctx, indexLookup, nil) + indexIter, err := index.RowIterForIndexLookup(ctx, indexLookup, t nil) require.NoError(t, err) var readRows []sql.Row diff --git a/go/libraries/doltcore/sqle/index/index_lookup.go b/go/libraries/doltcore/sqle/index/index_lookup.go index e21aa8175e..2adc935786 100644 --- a/go/libraries/doltcore/sqle/index/index_lookup.go +++ b/go/libraries/doltcore/sqle/index/index_lookup.go @@ -36,26 +36,35 @@ func PartitionIndexedTableRows(ctx *sql.Context, idx sql.Index, part sql.Partiti doltIdx := idx.(DoltIndex) if types.IsFormat_DOLT_1(rp.rows.Format()) { - covers := indexCoversCols(doltIdx, columns) - if covers { - return newProllyCoveringIndexIter(ctx, doltIdx, rp.prollyRange, pkSch) - } else { - return newProllyIndexIter(ctx, doltIdx, rp.prollyRange, columns) - } + return RowIterForProllyRange(ctx, doltIdx, rp.prollyRange, pkSch, columns) } ranges := []*noms.ReadRange{rp.nomsRange} - return RowIterForRanges(ctx, doltIdx, ranges, rp.rows, columns) + return RowIterForNomsRanges(ctx, doltIdx, ranges, rp.rows, columns) } -func RowIterForIndexLookup(ctx *sql.Context, ilu sql.IndexLookup, columns []string) (sql.RowIter, error) { +func RowIterForIndexLookup(ctx *sql.Context, ilu sql.IndexLookup, pkSch sql.PrimaryKeySchema, columns []string) (sql.RowIter, error) { lookup := ilu.(*doltIndexLookup) idx := lookup.idx - return RowIterForRanges(ctx, idx, lookup.nomsRanges, lookup.IndexRowData(), columns) + if types.IsFormat_DOLT_1(idx.Format()) { + // todo(andy) + return RowIterForProllyRange(ctx, idx, lookup.prollyRanges[0], pkSch, columns) + } else { + return RowIterForNomsRanges(ctx, idx, lookup.nomsRanges, lookup.IndexRowData(), columns) + } } -func RowIterForRanges(ctx *sql.Context, idx DoltIndex, ranges []*noms.ReadRange, rowData durable.Index, columns []string) (sql.RowIter, error) { +func RowIterForProllyRange(ctx *sql.Context, idx DoltIndex, ranges prolly.Range, pkSch sql.PrimaryKeySchema, columns []string) (sql.RowIter, error) { + covers := indexCoversCols(idx, columns) + if covers { + return newProllyCoveringIndexIter(ctx, idx, ranges, pkSch) + } else { + return newProllyIndexIter(ctx, idx, ranges) + } +} + +func RowIterForNomsRanges(ctx *sql.Context, idx DoltIndex, ranges []*noms.ReadRange, rowData durable.Index, columns []string) (sql.RowIter, error) { m := durable.NomsMapFromIndex(rowData) nrr := noms.NewNomsRangeReader(idx.IndexSchema(), m, ranges) diff --git a/go/libraries/doltcore/sqle/index/prolly_index_iter.go b/go/libraries/doltcore/sqle/index/prolly_index_iter.go index 0a793e7c5f..48b818ce78 100644 --- a/go/libraries/doltcore/sqle/index/prolly_index_iter.go +++ b/go/libraries/doltcore/sqle/index/prolly_index_iter.go @@ -49,7 +49,7 @@ type prollyIndexIter struct { var _ sql.RowIter = prollyIndexIter{} // NewProllyIndexIter returns a new prollyIndexIter. -func newProllyIndexIter(ctx *sql.Context, idx DoltIndex, rng prolly.Range, projection []string) (prollyIndexIter, error) { +func newProllyIndexIter(ctx *sql.Context, idx DoltIndex, rng prolly.Range) (prollyIndexIter, error) { secondary := durable.ProllyMapFromIndex(idx.IndexRowData()) indexIter, err := secondary.IterRange(ctx, rng) if err != nil { diff --git a/go/libraries/doltcore/sqle/indexed_dolt_table.go b/go/libraries/doltcore/sqle/indexed_dolt_table.go index 6a0c317d40..c7c8125f39 100644 --- a/go/libraries/doltcore/sqle/indexed_dolt_table.go +++ b/go/libraries/doltcore/sqle/indexed_dolt_table.go @@ -58,7 +58,7 @@ func (idt *IndexedDoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, er func (idt *IndexedDoltTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) { // todo(andy): only used by 'AS OF` queries - return index.RowIterForIndexLookup(ctx, idt.indexLookup, nil) + return index.RowIterForIndexLookup(ctx, idt.indexLookup, idt.table.sqlSch, nil) } func (idt *IndexedDoltTable) IsTemporary() bool { diff --git a/go/libraries/doltcore/sqle/procedures_table.go b/go/libraries/doltcore/sqle/procedures_table.go index f2592929ea..5b9d71abeb 100644 --- a/go/libraries/doltcore/sqle/procedures_table.go +++ b/go/libraries/doltcore/sqle/procedures_table.go @@ -172,7 +172,7 @@ func DoltProceduresGetDetails(ctx *sql.Context, tbl *WritableDoltTable, name str return sql.StoredProcedureDetails{}, false, err } - rowIter, err := index.RowIterForIndexLookup(ctx, indexLookup, nil) + rowIter, err := index.RowIterForIndexLookup(ctx, indexLookup, tbl.sqlSch, nil) if err != nil { return sql.StoredProcedureDetails{}, false, err } diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 00c4bd4d51..7e67da3887 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -18,6 +18,8 @@ import ( "fmt" "io" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" @@ -200,26 +202,41 @@ func nextSchemasTableIndex(ctx *sql.Context, root *doltdb.RootValue) (int64, err return 0, err } - rows, err := tbl.GetNomsRowData(ctx) + rows, err := tbl.GetRowData(ctx) if err != nil { return 0, err } - idx := int64(1) - if rows.Len() > 0 { - keyTpl, _, err := rows.Last(ctx) + if rows.Empty() { + return 1, nil + } + + if types.IsFormat_DOLT_1(tbl.Format()) { + p := durable.ProllyMapFromIndex(rows) + key, _, err := p.Last(ctx) if err != nil { return 0, err } - if keyTpl != nil { - key, err := keyTpl.(types.Tuple).Get(1) - if err != nil { - return 0, err - } - idx = int64(key.(types.Int)) + 1 + kd, _ := p.Descriptors() + + i, _ := kd.GetInt64(0, key) + return i + 1, nil + } else { + m := durable.NomsMapFromIndex(rows) + keyTpl, _, err := m.Last(ctx) + if err != nil { + return 0, err } + if keyTpl == nil { + return 1, nil + } + + key, err := keyTpl.(types.Tuple).Get(1) + if err != nil { + return 0, err + } + return int64(key.(types.Int)) + 1, nil } - return idx, nil } // fragFromSchemasTable returns the row with the given schema fragment if it exists. @@ -245,7 +262,7 @@ func fragFromSchemasTable(ctx *sql.Context, tbl *WritableDoltTable, fragType str return nil, false, err } - iter, err := index.RowIterForIndexLookup(ctx, lookup, nil) + iter, err := index.RowIterForIndexLookup(ctx, lookup, tbl.sqlSch, nil) if err != nil { return nil, false, err } @@ -271,7 +288,57 @@ type schemaFragment struct { fragment string } -func getSchemaFragmentsOfType(ctx *sql.Context, tbl *doltdb.Table, fragmentType string) ([]schemaFragment, error) { +func getSchemaFragmentsOfType(ctx *sql.Context, tbl *WritableDoltTable, fragType string) ([]schemaFragment, error) { + indexes, err := tbl.GetIndexes(ctx) + if err != nil { + return nil, err + } + var fragNameIndex sql.Index + for _, index := range indexes { + if index.ID() == doltdb.SchemasTablesIndexName { + fragNameIndex = index + break + } + } + if fragNameIndex == nil { + return nil, noSchemaIndexDefined + } + + exprs := fragNameIndex.Expressions() + lookup, err := sql.NewIndexBuilder(ctx, fragNameIndex).Equals(ctx, exprs[0], fragType).Build(ctx) + if err != nil { + return nil, err + } + + iter, err := index.RowIterForIndexLookup(ctx, lookup, tbl.sqlSch, nil) + if err != nil { + return nil, err + } + defer func() { + if cerr := iter.Close(ctx); cerr != nil { + err = cerr + } + }() + + var frags []schemaFragment + for { + sqlRow, err := iter.Next(ctx) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + frags = append(frags, schemaFragment{ + name: sqlRow[1].(string), + fragment: sqlRow[2].(string), + }) + } + return frags, nil +} + +func getSchemaFragmentsOfType2(ctx *sql.Context, tbl *doltdb.Table, fragmentType string) ([]schemaFragment, error) { sch, err := tbl.GetSchema(ctx) if err != nil { return nil, err diff --git a/go/store/prolly/map.go b/go/store/prolly/map.go index 66bb715033..47276bfdff 100644 --- a/go/store/prolly/map.go +++ b/go/store/prolly/map.go @@ -151,6 +151,19 @@ func (m Map) Has(ctx context.Context, key val.Tuple) (ok bool, err error) { return } +func (m Map) Last(ctx context.Context) (key, value val.Tuple, err error) { + cur, err := newCursorAtEnd(ctx, m.ns, m.root) + if err != nil { + return nil, nil, err + } + + if cur.valid() { + key = val.Tuple(cur.currentKey()) + value = val.Tuple(cur.currentValue()) + } + return +} + // IterAll returns a MutableMapRangeIter that iterates over the entire Map. func (m Map) IterAll(ctx context.Context) (MapRangeIter, error) { rng := Range{ diff --git a/go/store/prolly/node_cursor.go b/go/store/prolly/node_cursor.go index c77af6610a..92dabfc96b 100644 --- a/go/store/prolly/node_cursor.go +++ b/go/store/prolly/node_cursor.go @@ -62,7 +62,7 @@ func newCursorAtStart(ctx context.Context, nrw NodeStore, nd Node) (cur *nodeCur return } -func newCursorPastEnd(ctx context.Context, nrw NodeStore, nd Node) (cur *nodeCursor, err error) { +func newCursorAtEnd(ctx context.Context, nrw NodeStore, nd Node) (cur *nodeCursor, err error) { cur = &nodeCursor{nd: nd, nrw: nrw} cur.skipToNodeEnd() @@ -76,6 +76,14 @@ func newCursorPastEnd(ctx context.Context, nrw NodeStore, nd Node) (cur *nodeCur cur = &nodeCursor{nd: nd, parent: parent, nrw: nrw} cur.skipToNodeEnd() } + return +} + +func newCursorPastEnd(ctx context.Context, nrw NodeStore, nd Node) (cur *nodeCursor, err error) { + cur, err = newCursorAtEnd(ctx, nrw, nd) + if err != nil { + return nil, err + } // advance |cur| past the end ok, err := cur.advance(ctx) From e42de29ee4cadf947f58e71291fddf936c4b52eb Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 7 Mar 2022 16:08:12 -0800 Subject: [PATCH 4/8] enginetest run against new format without panicking --- go/libraries/doltcore/sqle/database.go | 67 +----------------- .../sqle/enginetest/dolt_engine_test.go | 3 + .../dolt_transaction_commit_test.go | 18 +++++ .../doltcore/sqle/procedures_table.go | 70 +++++++++++++++++++ go/libraries/doltcore/sqle/schema_table.go | 55 --------------- go/libraries/doltcore/sqle/tables.go | 20 ++++++ 6 files changed, 112 insertions(+), 121 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index be52a91da9..5a7b6dfffc 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -29,7 +29,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/env/actions/commitwalk" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/alterschema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" @@ -38,7 +37,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer" "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" - "github.com/dolthub/dolt/go/store/types" ) var ErrInvalidTableName = errors.NewKind("Invalid table name %s. Table names must match the regular expression " + doltdb.TableNameRegexStr) @@ -1027,70 +1025,7 @@ func (db Database) DropTrigger(ctx *sql.Context, name string) error { // GetStoredProcedures implements sql.StoredProcedureDatabase. func (db Database) GetStoredProcedures(ctx *sql.Context) ([]sql.StoredProcedureDetails, error) { - missingValue := errors.NewKind("missing `%s` value for procedure row: (%s)") - - root, err := db.GetRoot(ctx) - if err != nil { - return nil, err - } - - table, ok, err := root.GetTable(ctx, doltdb.ProceduresTableName) - if err != nil { - return nil, err - } - if !ok { - return nil, nil - } - - rowData, err := table.GetNomsRowData(ctx) - if err != nil { - return nil, err - } - - sch, err := table.GetSchema(ctx) - if err != nil { - return nil, err - } - - var spds []sql.StoredProcedureDetails - err = rowData.Iter(ctx, func(key, val types.Value) (stop bool, err error) { - dRow, err := row.FromNoms(sch, key.(types.Tuple), val.(types.Tuple)) - if err != nil { - return true, err - } - taggedVals, err := dRow.TaggedValues() - if err != nil { - return true, err - } - - name, ok := dRow.GetColVal(schema.DoltProceduresNameTag) - if !ok { - return true, missingValue.New(doltdb.ProceduresTableNameCol, taggedVals) - } - createStmt, ok := dRow.GetColVal(schema.DoltProceduresCreateStmtTag) - if !ok { - return true, missingValue.New(doltdb.ProceduresTableCreateStmtCol, taggedVals) - } - createdAt, ok := dRow.GetColVal(schema.DoltProceduresCreatedAtTag) - if !ok { - return true, missingValue.New(doltdb.ProceduresTableCreatedAtCol, taggedVals) - } - modifiedAt, ok := dRow.GetColVal(schema.DoltProceduresModifiedAtTag) - if !ok { - return true, missingValue.New(doltdb.ProceduresTableModifiedAtCol, taggedVals) - } - spds = append(spds, sql.StoredProcedureDetails{ - Name: string(name.(types.String)), - CreateStatement: string(createStmt.(types.String)), - CreatedAt: time.Time(createdAt.(types.Timestamp)), - ModifiedAt: time.Time(modifiedAt.(types.Timestamp)), - }) - return false, nil - }) - if err != nil { - return nil, err - } - return spds, nil + return DoltProceduresGetAll(ctx, db) } // SaveStoredProcedure implements sql.StoredProcedureDatabase. diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 4ff8d39135..28bab2de71 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -376,6 +376,9 @@ func TestStoredProcedures(t *testing.T) { } func TestTransactions(t *testing.T) { + if types.IsFormat_DOLT_1(types.Format_Default) { + t.Skip() + } enginetest.TestTransactionScripts(t, newDoltHarness(t)) for _, script := range DoltTransactionTests { enginetest.TestTransactionScript(t, newDoltHarness(t), script) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_transaction_commit_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_transaction_commit_test.go index c2944d96e1..bc296b562f 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_transaction_commit_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_transaction_commit_test.go @@ -18,6 +18,8 @@ import ( "context" "testing" + "github.com/dolthub/dolt/go/store/types" + "github.com/dolthub/go-mysql-server/enginetest" "github.com/dolthub/go-mysql-server/sql" "github.com/stretchr/testify/require" @@ -28,6 +30,10 @@ import ( // TODO: we need tests for manual DOLT_COMMIT as well, but that's difficult with the way that functions are resolved // in the engine. func TestDoltTransactionCommitOneClient(t *testing.T) { + if types.IsFormat_DOLT_1(types.Format_Default) { + t.Skip() + } + // In this test, we're setting only one client to match transaction commits to dolt commits. // Autocommit is disabled for the enabled client, as it's the recommended way to use this feature. harness := newDoltHarness(t) @@ -163,6 +169,10 @@ func TestDoltTransactionCommitOneClient(t *testing.T) { } func TestDoltTransactionCommitTwoClients(t *testing.T) { + if types.IsFormat_DOLT_1(types.Format_Default) { + t.Skip() + } + // In this test, we're setting both clients to match transaction commits to dolt commits. // Autocommit is disabled, as it's the recommended way to use this feature. harness := newDoltHarness(t) @@ -291,6 +301,10 @@ func TestDoltTransactionCommitTwoClients(t *testing.T) { } func TestDoltTransactionCommitAutocommit(t *testing.T) { + if types.IsFormat_DOLT_1(types.Format_Default) { + t.Skip() + } + // In this test, each insertion from both clients cause a commit as autocommit is enabled. // Not the recommended way to use the feature, but it's permitted. harness := newDoltHarness(t) @@ -365,6 +379,10 @@ func TestDoltTransactionCommitAutocommit(t *testing.T) { } func TestDoltTransactionCommitLateFkResolution(t *testing.T) { + if types.IsFormat_DOLT_1(types.Format_Default) { + t.Skip() + } + harness := newDoltHarness(t) enginetest.TestTransactionScript(t, harness, enginetest.TransactionTest{ Name: "delayed foreign key resolution with transaction commits", diff --git a/go/libraries/doltcore/sqle/procedures_table.go b/go/libraries/doltcore/sqle/procedures_table.go index 5b9d71abeb..0482485dbb 100644 --- a/go/libraries/doltcore/sqle/procedures_table.go +++ b/go/libraries/doltcore/sqle/procedures_table.go @@ -20,6 +20,8 @@ import ( "strings" "time" + "gopkg.in/src-d/go-errors.v1" + "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" @@ -95,6 +97,74 @@ func DoltProceduresGetTable(ctx *sql.Context, db Database) (*WritableDoltTable, return tbl.(*WritableDoltTable), nil } +func DoltProceduresGetAll(ctx *sql.Context, db Database) ([]sql.StoredProcedureDetails, error) { + tbl, err := DoltProceduresGetTable(ctx, db) + if err != nil { + return nil, err + } + + indexes, err := tbl.GetIndexes(ctx) + if err != nil { + return nil, err + } + if len(indexes) == 0 { + return nil, fmt.Errorf("missing index for stored procedures") + } + idx := indexes[0] + + if len(idx.Expressions()) == 0 { + return nil, fmt.Errorf("missing index expression for stored procedures") + } + nameExpr := idx.Expressions()[0] + + lookup, err := sql.NewIndexBuilder(ctx, idx).IsNotNull(ctx, nameExpr).Build(ctx) + if err != nil { + return nil, err + } + + iter, err := index.RowIterForIndexLookup(ctx, lookup, tbl.sqlSch, nil) + if err != nil { + return nil, err + } + defer func() { + if cerr := iter.Close(ctx); cerr != nil { + err = cerr + } + }() + + var sqlRow sql.Row + var details []sql.StoredProcedureDetails + missingValue := errors.NewKind("missing `%s` value for procedure row: (%s)") + + for { + sqlRow, err = iter.Next(ctx) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + var d sql.StoredProcedureDetails + var ok bool + + if d.Name, ok = sqlRow[0].(string); !ok { + return nil, missingValue.New(doltdb.ProceduresTableNameCol, sqlRow) + } + if d.CreateStatement, ok = sqlRow[1].(string); !ok { + return nil, missingValue.New(doltdb.ProceduresTableCreateStmtCol, sqlRow) + } + if d.CreatedAt, ok = sqlRow[2].(time.Time); !ok { + return nil, missingValue.New(doltdb.ProceduresTableCreatedAtCol, sqlRow) + } + if d.ModifiedAt, ok = sqlRow[3].(time.Time); !ok { + return nil, missingValue.New(doltdb.ProceduresTableModifiedAtCol, sqlRow) + } + details = append(details, d) + } + return details, nil +} + // DoltProceduresAddProcedure adds the stored procedure to the `dolt_procedures` table in the given db, creating it if // it does not exist. func DoltProceduresAddProcedure(ctx *sql.Context, db Database, spd sql.StoredProcedureDetails) (retErr error) { diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 7e67da3887..78d0bf4666 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -337,58 +337,3 @@ func getSchemaFragmentsOfType(ctx *sql.Context, tbl *WritableDoltTable, fragType } return frags, nil } - -func getSchemaFragmentsOfType2(ctx *sql.Context, tbl *doltdb.Table, fragmentType string) ([]schemaFragment, error) { - sch, err := tbl.GetSchema(ctx) - if err != nil { - return nil, err - } - - typeCol, ok := sch.GetAllCols().GetByName(doltdb.SchemasTablesTypeCol) - if !ok { - return nil, errDoltSchemasTableFormat - } - nameCol, ok := sch.GetAllCols().GetByName(doltdb.SchemasTablesNameCol) - if !ok { - return nil, errDoltSchemasTableFormat - } - fragCol, ok := sch.GetAllCols().GetByName(doltdb.SchemasTablesFragmentCol) - if !ok { - return nil, errDoltSchemasTableFormat - } - - rowData, err := tbl.GetNomsRowData(ctx) - if err != nil { - return nil, err - } - - var fragments []schemaFragment - err = rowData.Iter(ctx, func(key, val types.Value) (stop bool, err error) { - dRow, err := row.FromNoms(sch, key.(types.Tuple), val.(types.Tuple)) - if err != nil { - return true, err - } - if typeColVal, ok := dRow.GetColVal(typeCol.Tag); ok && typeColVal.Equals(types.String(fragmentType)) { - name, ok := dRow.GetColVal(nameCol.Tag) - if !ok { - taggedVals, _ := dRow.TaggedValues() - return true, fmt.Errorf("missing `%s` value for view row: (%s)", doltdb.SchemasTablesNameCol, taggedVals) - } - def, ok := dRow.GetColVal(fragCol.Tag) - if !ok { - taggedVals, _ := dRow.TaggedValues() - return true, fmt.Errorf("missing `%s` value for view row: (%s)", doltdb.SchemasTablesFragmentCol, taggedVals) - } - fragments = append(fragments, schemaFragment{ - name: string(name.(types.String)), - fragment: string(def.(types.String)), - }) - } - return false, nil - }) - if err != nil { - return nil, err - } - - return fragments, nil -} diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index d933994559..158c974e0f 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -864,6 +864,10 @@ var _ doltAlterableTableInterface = (*AlterableDoltTable)(nil) // AddColumn implements sql.AlterableTable func (t *AlterableDoltTable) AddColumn(ctx *sql.Context, column *sql.Column, order *sql.ColumnOrder) error { + if types.IsFormat_DOLT_1(t.nbf) { + return nil + } + root, err := t.getRoot(ctx) if err != nil { @@ -928,6 +932,10 @@ func orderToOrder(order *sql.ColumnOrder) *alterschema.ColumnOrder { // DropColumn implements sql.AlterableTable func (t *AlterableDoltTable) DropColumn(ctx *sql.Context, columnName string) error { + if types.IsFormat_DOLT_1(t.nbf) { + return nil + } + root, err := t.getRoot(ctx) if err != nil { return err @@ -1050,6 +1058,10 @@ func (t *AlterableDoltTable) dropColumnData(ctx *sql.Context, updatedTable *dolt // ModifyColumn implements sql.AlterableTable func (t *AlterableDoltTable) ModifyColumn(ctx *sql.Context, columnName string, column *sql.Column, order *sql.ColumnOrder) error { + if types.IsFormat_DOLT_1(t.nbf) { + return nil + } + root, err := t.getRoot(ctx) if err != nil { return err @@ -1859,6 +1871,10 @@ func (t *AlterableDoltTable) constraintNameExists(ctx *sql.Context, name string) } func (t *AlterableDoltTable) CreatePrimaryKey(ctx *sql.Context, columns []sql.IndexColumn) error { + if types.IsFormat_DOLT_1(t.nbf) { + return nil + } + table, err := t.doltTable(ctx) if err != nil { return err @@ -1889,6 +1905,10 @@ func (t *AlterableDoltTable) CreatePrimaryKey(ctx *sql.Context, columns []sql.In } func (t *AlterableDoltTable) DropPrimaryKey(ctx *sql.Context) error { + if types.IsFormat_DOLT_1(t.nbf) { + return nil + } + // Ensure that no auto increment requirements exist on this table if t.autoIncCol.AutoIncrement { return sql.ErrWrongAutoKey.New() From 89a805ea95bd8a89bf6977fe5ff803a37db5b6b9 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 7 Mar 2022 16:16:53 -0800 Subject: [PATCH 5/8] cleanup --- .../doltcore/sqle/enginetest/dolt_transaction_commit_test.go | 3 +-- go/libraries/doltcore/sqle/schema_table.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_transaction_commit_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_transaction_commit_test.go index bc296b562f..50deb1bb35 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_transaction_commit_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_transaction_commit_test.go @@ -18,13 +18,12 @@ import ( "context" "testing" - "github.com/dolthub/dolt/go/store/types" - "github.com/dolthub/go-mysql-server/enginetest" "github.com/dolthub/go-mysql-server/sql" "github.com/stretchr/testify/require" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/store/types" ) // TODO: we need tests for manual DOLT_COMMIT as well, but that's difficult with the way that functions are resolved diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 78d0bf4666..12af9e8e5a 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -18,11 +18,11 @@ import ( "fmt" "io" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" From bf247a2a274523c434e77c2d9d1e41db8e130202 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 7 Mar 2022 16:22:46 -0800 Subject: [PATCH 6/8] repofmt --- go/libraries/doltcore/sqle/index/dolt_index_test.go | 9 ++++++--- go/libraries/doltcore/sqle/procedures_table.go | 3 +-- go/libraries/doltcore/sqle/schema_table.go | 1 - 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/go/libraries/doltcore/sqle/index/dolt_index_test.go b/go/libraries/doltcore/sqle/index/dolt_index_test.go index 8f891bfb88..1949c494b1 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index_test.go +++ b/go/libraries/doltcore/sqle/index/dolt_index_test.go @@ -18,7 +18,6 @@ import ( "context" "errors" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "io" "sort" "strings" @@ -32,6 +31,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/sqle" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/libraries/utils/config" ) @@ -1254,7 +1254,7 @@ func requireUnorderedRowsEqual(t *testing.T, rows1, rows2 []sql.Row) { require.Equal(t, rows1, rows2) } -func testDoltIndex(t *testing.T, keys []interface{}, expectedRows []sql.Row, idx sql.Index, cmp indexComp) { +func testDoltIndex(t *testing.T, keys []interface{}, expectedRows []sql.Row, idx index.DoltIndex, cmp indexComp) { ctx := NewTestSQLCtx(context.Background()) exprs := idx.Expressions() builder := sql.NewIndexBuilder(sql.NewEmptyContext(), idx) @@ -1279,7 +1279,10 @@ func testDoltIndex(t *testing.T, keys []interface{}, expectedRows []sql.Row, idx indexLookup, err := builder.Build(ctx) require.NoError(t, err) - indexIter, err := index.RowIterForIndexLookup(ctx, indexLookup, t nil) + pkSch, err := sqlutil.FromDoltSchema("fake_table", idx.Schema()) + require.NoError(t, err) + + indexIter, err := index.RowIterForIndexLookup(ctx, indexLookup, pkSch, nil) require.NoError(t, err) var readRows []sql.Row diff --git a/go/libraries/doltcore/sqle/procedures_table.go b/go/libraries/doltcore/sqle/procedures_table.go index 0482485dbb..d29fb91fcf 100644 --- a/go/libraries/doltcore/sqle/procedures_table.go +++ b/go/libraries/doltcore/sqle/procedures_table.go @@ -20,9 +20,8 @@ import ( "strings" "time" - "gopkg.in/src-d/go-errors.v1" - "github.com/dolthub/go-mysql-server/sql" + "gopkg.in/src-d/go-errors.v1" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/schema" diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 12af9e8e5a..887c75bb25 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -18,7 +18,6 @@ import ( "fmt" "io" - "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" From 2a66864bcc3ec799bb215f7f2f02044eb9bd4cda Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Tue, 8 Mar 2022 14:09:40 -0800 Subject: [PATCH 7/8] fix trigger and stored procedure tests --- go/libraries/doltcore/sqle/database.go | 22 +++++++++---- go/libraries/doltcore/sqle/rows.go | 10 ++++-- go/libraries/doltcore/sqle/schema_table.go | 36 ++++++---------------- 3 files changed, 32 insertions(+), 36 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 5a7b6dfffc..58a1218bdf 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -923,12 +923,15 @@ func (db Database) GetView(ctx *sql.Context, viewName string) (string, bool, err return view, true, nil } - tbl, err := GetOrCreateDoltSchemasTable(ctx, db) + tbl, ok, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return "", false, err } + if !ok { + return "", false, nil + } - fragments, err := getSchemaFragmentsOfType(ctx, tbl, "view") + fragments, err := getSchemaFragmentsOfType(ctx, tbl.(*WritableDoltTable), viewFragment) if err != nil { return "", false, err } @@ -944,12 +947,15 @@ func (db Database) GetView(ctx *sql.Context, viewName string) (string, bool, err // GetView implements sql.ViewDatabase func (db Database) AllViews(ctx *sql.Context) ([]sql.ViewDefinition, error) { - tbl, err := GetOrCreateDoltSchemasTable(ctx, db) + tbl, ok, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return nil, err } + if !ok { + return nil, nil + } - frags, err := getSchemaFragmentsOfType(ctx, tbl, "view") + frags, err := getSchemaFragmentsOfType(ctx, tbl.(*WritableDoltTable), viewFragment) if err != nil { return nil, err } @@ -984,11 +990,15 @@ func (db Database) DropView(ctx *sql.Context, name string) error { // GetTriggers implements sql.TriggerDatabase. func (db Database) GetTriggers(ctx *sql.Context) ([]sql.TriggerDefinition, error) { - tbl, err := GetOrCreateDoltSchemasTable(ctx, db) + tbl, ok, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return nil, err } - frags, err := getSchemaFragmentsOfType(ctx, tbl, "trigger") + if !ok { + return nil, nil + } + + frags, err := getSchemaFragmentsOfType(ctx, tbl.(*WritableDoltTable), triggerFragment) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/rows.go b/go/libraries/doltcore/sqle/rows.go index b7878a2c26..26c6195551 100644 --- a/go/libraries/doltcore/sqle/rows.go +++ b/go/libraries/doltcore/sqle/rows.go @@ -184,10 +184,14 @@ func ProllyRowIterFromPartition(ctx context.Context, tbl *doltdb.Table, projecti // returned rows always have the schema of the table, regardless of the value // of |columns|. Providing a column name which does not appear in the schema // is not an error, but no corresponding column will appear in the results. -func TableToRowIter(ctx context.Context, table *doltdb.Table, columns []string) (sql.RowIter, error) { - data, err := table.GetRowData(ctx) +func TableToRowIter(ctx *sql.Context, table *WritableDoltTable, columns []string) (sql.RowIter, error) { + t, err := table.doltTable(ctx) if err != nil { return nil, err } - return newRowIterator(ctx, table, columns, doltTablePartition{rowData: data, end: NoUpperBound}) + data, err := t.GetRowData(ctx) + if err != nil { + return nil, err + } + return newRowIterator(ctx, t, columns, doltTablePartition{rowData: data, end: NoUpperBound}) } diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 887c75bb25..dd8a7e72ec 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -33,6 +33,11 @@ import ( var errDoltSchemasTableFormat = fmt.Errorf("`%s` schema in unexpected format", doltdb.SchemasTableName) var noSchemaIndexDefined = fmt.Errorf("could not find index `%s` on system table `%s`", doltdb.SchemasTablesIndexName, doltdb.SchemasTableName) +const ( + viewFragment = "view" + triggerFragment = "trigger" +) + // The fixed dolt schema for the `dolt_schemas` table. func SchemasTableSchema() schema.Schema { typeCol, err := schema.NewColumnWithTypeInfo(doltdb.SchemasTablesTypeCol, schema.DoltSchemasTypeTag, typeinfo.LegacyStringDefaultType, false, "", false, "") @@ -288,36 +293,10 @@ type schemaFragment struct { } func getSchemaFragmentsOfType(ctx *sql.Context, tbl *WritableDoltTable, fragType string) ([]schemaFragment, error) { - indexes, err := tbl.GetIndexes(ctx) + iter, err := TableToRowIter(ctx, tbl, nil) if err != nil { return nil, err } - var fragNameIndex sql.Index - for _, index := range indexes { - if index.ID() == doltdb.SchemasTablesIndexName { - fragNameIndex = index - break - } - } - if fragNameIndex == nil { - return nil, noSchemaIndexDefined - } - - exprs := fragNameIndex.Expressions() - lookup, err := sql.NewIndexBuilder(ctx, fragNameIndex).Equals(ctx, exprs[0], fragType).Build(ctx) - if err != nil { - return nil, err - } - - iter, err := index.RowIterForIndexLookup(ctx, lookup, tbl.sqlSch, nil) - if err != nil { - return nil, err - } - defer func() { - if cerr := iter.Close(ctx); cerr != nil { - err = cerr - } - }() var frags []schemaFragment for { @@ -329,6 +308,9 @@ func getSchemaFragmentsOfType(ctx *sql.Context, tbl *WritableDoltTable, fragType return nil, err } + if sqlRow[0] != fragType { + continue + } frags = append(frags, schemaFragment{ name: sqlRow[1].(string), fragment: sqlRow[2].(string), From 2b8a9c2621e260eeea239030f8ad3e7003aab0ab Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Tue, 8 Mar 2022 14:19:22 -0800 Subject: [PATCH 8/8] fix TableToRowIter for prolly maps --- go/libraries/doltcore/sqle/rows.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/rows.go b/go/libraries/doltcore/sqle/rows.go index 26c6195551..c5fae56377 100644 --- a/go/libraries/doltcore/sqle/rows.go +++ b/go/libraries/doltcore/sqle/rows.go @@ -25,6 +25,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" "github.com/dolthub/dolt/go/libraries/doltcore/table" "github.com/dolthub/dolt/go/libraries/utils/set" + "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/types" ) @@ -193,5 +194,21 @@ func TableToRowIter(ctx *sql.Context, table *WritableDoltTable, columns []string if err != nil { return nil, err } - return newRowIterator(ctx, t, columns, doltTablePartition{rowData: data, end: NoUpperBound}) + + p := doltTablePartition{ + end: NoUpperBound, + rowData: data, + } + + if types.IsFormat_DOLT_1(data.Format()) { + m := durable.ProllyMapFromIndex(data) + kd, _ := m.Descriptors() + p.rowRange = prolly.Range{ + Start: prolly.RangeCut{Unbound: true}, + Stop: prolly.RangeCut{Unbound: true}, + KeyDesc: kd, + } + } + + return newRowIterator(ctx, t, columns, p) }