From 371c56c39d9469898f9b1a4a23567128448c4f91 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 19 May 2020 15:29:11 -0700 Subject: [PATCH 1/7] Working modifications for index refactoring Signed-off-by: Zach Musgrave --- go/go.mod | 3 +++ go/libraries/doltcore/sqle/index_driver.go | 10 +++++----- go/libraries/doltcore/sqle/index_lookup.go | 12 ++++++++---- go/libraries/doltcore/sqle/indexed_dolt_table.go | 13 ++++--------- go/libraries/doltcore/sqle/tables.go | 6 +++--- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/go/go.mod b/go/go.mod index 6d02cca235..e06bf10bd6 100644 --- a/go/go.mod +++ b/go/go.mod @@ -52,6 +52,7 @@ require ( github.com/mattn/go-runewidth v0.0.9 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b github.com/miekg/dns v1.1.27 // indirect + github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.6.0 // indirect github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.4.0 @@ -89,6 +90,8 @@ require ( replace github.com/liquidata-inc/dolt/go/gen/proto/dolt/services/eventsapi => ./gen/proto/dolt/services/eventsapi +replace github.com/liquidata-inc/go-mysql-server => ../../go-mysql-server + replace vitess.io/vitess => github.com/liquidata-inc/vitess v0.0.0-20200430040751-192bb76ecd8b go 1.13 diff --git a/go/libraries/doltcore/sqle/index_driver.go b/go/libraries/doltcore/sqle/index_driver.go index 1820de9b41..3da06ab3c5 100644 --- a/go/libraries/doltcore/sqle/index_driver.go +++ b/go/libraries/doltcore/sqle/index_driver.go @@ -37,11 +37,11 @@ func NewDoltIndexDriver(dbs ...Database) *DoltIndexDriver { return &DoltIndexDriver{nameToDB} } -func (*DoltIndexDriver) Create(string, string, string, []sql.Expression, map[string]string) (sql.Index, error) { +func (*DoltIndexDriver) Create(string, string, string, []sql.Expression, map[string]string) (sql.DriverIndex, error) { panic("index driver create path not supported") } -func (i *DoltIndexDriver) Delete(sql.Index, sql.PartitionIter) error { +func (i *DoltIndexDriver) Delete(sql.DriverIndex, sql.PartitionIter) error { panic("index driver delete path not supported") } @@ -49,7 +49,7 @@ func (*DoltIndexDriver) ID() string { return "doltDbIndexDriver" } -func (driver *DoltIndexDriver) LoadAll(ctx *sql.Context, db, table string) ([]sql.Index, error) { +func (driver *DoltIndexDriver) LoadAll(ctx *sql.Context, db, table string) ([]sql.DriverIndex, error) { database, ok := driver.dbs[db] if !ok { panic("Unexpected db: " + db) @@ -79,7 +79,7 @@ func (driver *DoltIndexDriver) LoadAll(ctx *sql.Context, db, table string) ([]sq } cols := sch.GetPKCols().GetColumns() - sqlIndexes := []sql.Index{ + sqlIndexes := []sql.DriverIndex{ &doltIndex{ cols: cols, ctx: ctx, @@ -121,6 +121,6 @@ func (driver *DoltIndexDriver) LoadAll(ctx *sql.Context, db, table string) ([]sq return sqlIndexes, nil } -func (i *DoltIndexDriver) Save(*sql.Context, sql.Index, sql.PartitionIndexKeyValueIter) error { +func (i *DoltIndexDriver) Save(*sql.Context, sql.DriverIndex, sql.PartitionIndexKeyValueIter) error { panic("index driver save path not supported") } diff --git a/go/libraries/doltcore/sqle/index_lookup.go b/go/libraries/doltcore/sqle/index_lookup.go index 23b325645d..c0a46181fd 100644 --- a/go/libraries/doltcore/sqle/index_lookup.go +++ b/go/libraries/doltcore/sqle/index_lookup.go @@ -15,6 +15,7 @@ package sqle import ( + "fmt" "github.com/liquidata-inc/go-mysql-server/sql" "github.com/liquidata-inc/dolt/go/libraries/doltcore/row" @@ -31,12 +32,15 @@ type doltIndexLookup struct { keyIter IndexLookupKeyIterator } -func (il *doltIndexLookup) Indexes() []string { - return []string{il.idx.ID()} +func (il *doltIndexLookup) String() string { + // TODO: fix + return fmt.Sprintf("%s:%s", il.idx.ID(), "") } -func (il *doltIndexLookup) Values(p sql.Partition) (sql.IndexValueIter, error) { - panic("implement me") +var _ sql.IndexLookup = (*doltIndexLookup)(nil) + +func (il *doltIndexLookup) Indexes() []string { + return []string{il.idx.ID()} } // RowIter returns a row iterator for this index lookup. The iterator will return the single matching row for the index. diff --git a/go/libraries/doltcore/sqle/indexed_dolt_table.go b/go/libraries/doltcore/sqle/indexed_dolt_table.go index 906faaa653..48cea8d4a3 100644 --- a/go/libraries/doltcore/sqle/indexed_dolt_table.go +++ b/go/libraries/doltcore/sqle/indexed_dolt_table.go @@ -24,12 +24,11 @@ type IndexedDoltTable struct { indexLookup *doltIndexLookup } -func (idt *IndexedDoltTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table { - return idt.table.WithIndexLookup(lookup) -} +var _ sql.IndexedTable = (*IndexedDoltTable)(nil) -func (idt *IndexedDoltTable) IndexKeyValues(*sql.Context, []string) (sql.PartitionIndexKeyValueIter, error) { - return idt.table.IndexKeyValues(nil, nil) +func (idt *IndexedDoltTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table { + // TODO: this should probably be an error (there should be at most one indexed lookup on a given table) + return idt.table.WithIndexLookup(lookup) } func (idt *IndexedDoltTable) Name() string { @@ -44,10 +43,6 @@ func (idt *IndexedDoltTable) Schema() sql.Schema { return idt.table.Schema() } -func (idt *IndexedDoltTable) IndexLookup() sql.IndexLookup { - return idt.indexLookup -} - func (idt *IndexedDoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) { return idt.table.Partitions(ctx) } diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index 0c54d966e1..8d13b864e1 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -39,10 +39,10 @@ type DoltTable struct { } var _ sql.Table = (*DoltTable)(nil) -var _ sql.IndexableTable = (*DoltTable)(nil) +var _ sql.IndexedTable = (*DoltTable)(nil) var _ sql.IndexAlterableTable = (*DoltTable)(nil) -// Implements sql.IndexableTable +// Implements sql.IndexedTable func (t *DoltTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table { dil, ok := lookup.(*doltIndexLookup) if !ok { @@ -61,7 +61,7 @@ func (t *DoltTable) IndexKeyValues(*sql.Context, []string) (sql.PartitionIndexKe } // Implements sql.IndexableTable -func (t *DoltTable) IndexLookup() sql.IndexLookup { +func (t *DoltTable) IndexLookup() sql.DriverIndexLookup { panic("IndexLookup called on DoltTable, should be on IndexedDoltTable") } From 76b9344f2c2166d78b5c85d6503abca02fa70e21 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 22 May 2020 14:00:25 -0700 Subject: [PATCH 2/7] Work in progress for index changes Signed-off-by: Zach Musgrave --- go/libraries/doltcore/sqle/indexed_dolt_table.go | 5 +++++ go/libraries/doltcore/sqle/tables.go | 11 +++-------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go/libraries/doltcore/sqle/indexed_dolt_table.go b/go/libraries/doltcore/sqle/indexed_dolt_table.go index 48cea8d4a3..4a2960fb81 100644 --- a/go/libraries/doltcore/sqle/indexed_dolt_table.go +++ b/go/libraries/doltcore/sqle/indexed_dolt_table.go @@ -26,6 +26,11 @@ type IndexedDoltTable struct { var _ sql.IndexedTable = (*IndexedDoltTable)(nil) +func (idt *IndexedDoltTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) { + // TODO: move index driver implementation here + panic("implement me") +} + func (idt *IndexedDoltTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table { // TODO: this should probably be an error (there should be at most one indexed lookup on a given table) return idt.table.WithIndexLookup(lookup) diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index 8d13b864e1..7489118603 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -55,14 +55,9 @@ func (t *DoltTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table { } } -// Implements sql.IndexableTable -func (t *DoltTable) IndexKeyValues(*sql.Context, []string) (sql.PartitionIndexKeyValueIter, error) { - return nil, errors.New("creating new indexes not supported") -} - -// Implements sql.IndexableTable -func (t *DoltTable) IndexLookup() sql.DriverIndexLookup { - panic("IndexLookup called on DoltTable, should be on IndexedDoltTable") +func (t *DoltTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) { + // TODO: move driver implementation here + panic("implement me") } // Name returns the name of the table. From 018a3838ccb0f59c575e24320057fc872003c016 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 22 May 2020 14:26:56 -0700 Subject: [PATCH 3/7] Killed index driver and related unnecessary methods Signed-off-by: Zach Musgrave --- go/cmd/dolt/commands/sql.go | 6 - go/cmd/dolt/commands/sqlserver/server.go | 8 - go/libraries/doltcore/sqle/dolt_index.go | 6 - go/libraries/doltcore/sqle/dolt_index_test.go | 8 - go/libraries/doltcore/sqle/index_driver.go | 126 -------- go/libraries/doltcore/sqle/index_lookup.go | 4 - .../sqle/logictest/dolt/doltharness.go | 7 - go/libraries/doltcore/sqle/tables.go | 298 ++++++++++-------- go/libraries/doltcore/sqle/testutil.go | 7 - 9 files changed, 175 insertions(+), 295 deletions(-) delete mode 100644 go/libraries/doltcore/sqle/index_driver.go diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index 1b9e6a9533..b730206415 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -1054,12 +1054,6 @@ func newSqlEngine(sqlCtx *sql.Context, mrEnv env.MultiRepoEnv, roots map[string] } } - sqlCtx.RegisterIndexDriver(dsqle.NewDoltIndexDriver(dbs...)) - err := sqlCtx.LoadIndexes(sqlCtx, engine.Catalog.AllDatabases()) - if err != nil { - return nil, err - } - return &sqlEngine{nameToDB, mrEnv, engine, format}, nil } diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index 0753eb7a1e..e2881fad89 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -186,14 +186,6 @@ func newSessionBuilder(sqlEngine *sqle.Engine, username, email string, autocommi } } - // TODO: this shouldn't need to happen every session - sqlCtx.RegisterIndexDriver(dsqle.NewDoltIndexDriver(dbs...)) - err = ir.LoadIndexes(sqlCtx, sqlEngine.Catalog.AllDatabases()) - - if err != nil { - return nil, nil, nil, err - } - return doltSess, ir, vr, nil } } diff --git a/go/libraries/doltcore/sqle/dolt_index.go b/go/libraries/doltcore/sqle/dolt_index.go index ccf678ebdc..b89d00bb57 100644 --- a/go/libraries/doltcore/sqle/dolt_index.go +++ b/go/libraries/doltcore/sqle/dolt_index.go @@ -37,9 +37,7 @@ type DoltIndex interface { type doltIndex struct { cols []schema.Column - ctx *sql.Context db Database - driver *DoltIndexDriver id string indexRowData types.Map indexSch schema.Schema @@ -121,10 +119,6 @@ func (di *doltIndex) DoltDatabase() Database { return di.db } -func (di *doltIndex) Driver() string { - return di.driver.ID() -} - func (di *doltIndex) Expressions() []string { strs := make([]string, len(di.cols)) for i, col := range di.cols { diff --git a/go/libraries/doltcore/sqle/dolt_index_test.go b/go/libraries/doltcore/sqle/dolt_index_test.go index 458a47d3d0..009a4c9479 100644 --- a/go/libraries/doltcore/sqle/dolt_index_test.go +++ b/go/libraries/doltcore/sqle/dolt_index_test.go @@ -1056,9 +1056,7 @@ INSERT INTO types VALUES (1, 4, '2020-05-14 12:00:03', 1.1, 'd', 1.1, 'a,c', '00 indexMap := map[string]DoltIndex{ "onepk:primaryKey": &doltIndex{ cols: tableSchemaMap["onepk"].GetPKCols().GetColumns(), - ctx: ctx, db: db, - driver: nil, id: "onepk:primaryKey", indexRowData: tableDataMap["onepk"], indexSch: tableSchemaMap["onepk"], @@ -1069,9 +1067,7 @@ INSERT INTO types VALUES (1, 4, '2020-05-14 12:00:03', 1.1, 'd', 1.1, 'a,c', '00 }, "twopk:primaryKey": &doltIndex{ cols: tableSchemaMap["twopk"].GetPKCols().GetColumns(), - ctx: ctx, db: db, - driver: nil, id: "twopk:primaryKey", indexRowData: tableDataMap["twopk"], indexSch: tableSchemaMap["twopk"], @@ -1082,9 +1078,7 @@ INSERT INTO types VALUES (1, 4, '2020-05-14 12:00:03', 1.1, 'd', 1.1, 'a,c', '00 }, "types:primaryKey": &doltIndex{ cols: tableSchemaMap["types"].GetPKCols().GetColumns(), - ctx: ctx, db: db, - driver: nil, id: "types:primaryKey", indexRowData: tableDataMap["types"], indexSch: tableSchemaMap["types"], @@ -1155,9 +1149,7 @@ INSERT INTO types VALUES (1, 4, '2020-05-14 12:00:03', 1.1, 'd', 1.1, 'a,c', '00 indexId := indexDetails.tableName + ":" + index.Name() indexMap[indexId] = &doltIndex{ cols: indexCols, - ctx: ctx, db: db, - driver: nil, id: indexId, indexRowData: indexData, indexSch: index.Schema(), diff --git a/go/libraries/doltcore/sqle/index_driver.go b/go/libraries/doltcore/sqle/index_driver.go deleted file mode 100644 index 3da06ab3c5..0000000000 --- a/go/libraries/doltcore/sqle/index_driver.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2020 Liquidata, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqle - -import ( - "fmt" - - "github.com/liquidata-inc/go-mysql-server/sql" - - "github.com/liquidata-inc/dolt/go/libraries/doltcore/schema" -) - -type DoltIndexDriver struct { - dbs map[string]Database -} - -var _ sql.IndexDriver = (*DoltIndexDriver)(nil) - -func NewDoltIndexDriver(dbs ...Database) *DoltIndexDriver { - nameToDB := make(map[string]Database) - for _, db := range dbs { - nameToDB[db.Name()] = db - } - - return &DoltIndexDriver{nameToDB} -} - -func (*DoltIndexDriver) Create(string, string, string, []sql.Expression, map[string]string) (sql.DriverIndex, error) { - panic("index driver create path not supported") -} - -func (i *DoltIndexDriver) Delete(sql.DriverIndex, sql.PartitionIter) error { - panic("index driver delete path not supported") -} - -func (*DoltIndexDriver) ID() string { - return "doltDbIndexDriver" -} - -func (driver *DoltIndexDriver) LoadAll(ctx *sql.Context, db, table string) ([]sql.DriverIndex, error) { - database, ok := driver.dbs[db] - if !ok { - panic("Unexpected db: " + db) - } - - root, err := database.GetRoot(ctx) - if err != nil { - return nil, err - } - - tbl, ok, err := root.GetTable(ctx, table) - if err != nil { - return nil, err - } - if !ok { - return nil, nil - } - - sch, err := tbl.GetSchema(ctx) - if err != nil { - return nil, err - } - - rowData, err := tbl.GetRowData(ctx) - if err != nil { - return nil, err - } - - cols := sch.GetPKCols().GetColumns() - sqlIndexes := []sql.DriverIndex{ - &doltIndex{ - cols: cols, - ctx: ctx, - db: database, - driver: driver, - id: fmt.Sprintf("%s:primaryKey%v", table, len(cols)), - indexRowData: rowData, - indexSch: sch, - table: tbl, - tableData: rowData, - tableName: table, - tableSch: sch, - }, - } - for _, index := range sch.Indexes().AllIndexes() { - indexRowData, err := tbl.GetIndexRowData(ctx, index.Name()) - if err != nil { - return nil, err - } - cols := make([]schema.Column, index.Count()) - for i, tag := range index.IndexedColumnTags() { - cols[i], _ = index.GetColumn(tag) - } - sqlIndexes = append(sqlIndexes, &doltIndex{ - cols: cols, - ctx: ctx, - db: database, - driver: driver, - id: table + ":" + index.Name(), - indexRowData: indexRowData, - indexSch: index.Schema(), - table: tbl, - tableData: rowData, - tableName: table, - tableSch: sch, - }) - } - - return sqlIndexes, nil -} - -func (i *DoltIndexDriver) Save(*sql.Context, sql.DriverIndex, sql.PartitionIndexKeyValueIter) error { - panic("index driver save path not supported") -} diff --git a/go/libraries/doltcore/sqle/index_lookup.go b/go/libraries/doltcore/sqle/index_lookup.go index c0a46181fd..2b8ecdfdbf 100644 --- a/go/libraries/doltcore/sqle/index_lookup.go +++ b/go/libraries/doltcore/sqle/index_lookup.go @@ -39,10 +39,6 @@ func (il *doltIndexLookup) String() string { var _ sql.IndexLookup = (*doltIndexLookup)(nil) -func (il *doltIndexLookup) Indexes() []string { - return []string{il.idx.ID()} -} - // RowIter returns a row iterator for this index lookup. The iterator will return the single matching row for the index. func (il *doltIndexLookup) RowIter(ctx *sql.Context) (sql.RowIter, error) { return &indexLookupRowIterAdapter{indexLookup: il, ctx: ctx}, nil diff --git a/go/libraries/doltcore/sqle/logictest/dolt/doltharness.go b/go/libraries/doltcore/sqle/logictest/dolt/doltharness.go index e0332e08a8..f28442c759 100644 --- a/go/libraries/doltcore/sqle/logictest/dolt/doltharness.go +++ b/go/libraries/doltcore/sqle/logictest/dolt/doltharness.go @@ -164,17 +164,10 @@ func innerInit(h *DoltHarness, dEnv *env.DoltEnv) error { } } - ctx.RegisterIndexDriver(dsql.NewDoltIndexDriver(dsqlDBs...)) - err = ctx.LoadIndexes(ctx, h.engine.Catalog.AllDatabases()) - if len(dbs) == 1 { h.sess.SetCurrentDatabase(dbs[0].Name()) } - if err != nil { - return err - } - return nil } diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index 7489118603..06566f1da3 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -40,9 +40,8 @@ type DoltTable struct { var _ sql.Table = (*DoltTable)(nil) var _ sql.IndexedTable = (*DoltTable)(nil) -var _ sql.IndexAlterableTable = (*DoltTable)(nil) -// Implements sql.IndexedTable +// WithIndexLookup implements sql.IndexedTable func (t *DoltTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table { dil, ok := lookup.(*doltIndexLookup) if !ok { @@ -55,9 +54,58 @@ func (t *DoltTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table { } } +// GetIndexes implements sql.IndexedTable func (t *DoltTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) { - // TODO: move driver implementation here - panic("implement me") + tbl := t.table + + sch, err := tbl.GetSchema(ctx) + if err != nil { + return nil, err + } + + rowData, err := tbl.GetRowData(ctx) + if err != nil { + return nil, err + } + + cols := sch.GetPKCols().GetColumns() + sqlIndexes := []sql.Index{ + &doltIndex{ + cols: cols, + db: t.db, + id: fmt.Sprintf("%s:primaryKey%v", t.Name(), len(cols)), + indexRowData: rowData, + indexSch: sch, + table: tbl, + tableData: rowData, + tableName: t.Name(), + tableSch: sch, + }, + } + + for _, index := range sch.Indexes().AllIndexes() { + indexRowData, err := tbl.GetIndexRowData(ctx, index.Name()) + if err != nil { + return nil, err + } + cols := make([]schema.Column, index.Count()) + for i, tag := range index.IndexedColumnTags() { + cols[i], _ = index.GetColumn(tag) + } + sqlIndexes = append(sqlIndexes, &doltIndex{ + cols: cols, + db: t.db, + id: index.Name(), + indexRowData: indexRowData, + indexSch: index.Schema(), + table: tbl, + tableData: rowData, + tableName: t.Name(), + tableSch: sch, + }) + } + + return sqlIndexes, nil } // Name returns the name of the table. @@ -65,7 +113,7 @@ func (t *DoltTable) Name() string { return t.name } -// Not sure what the purpose of this method is, so returning the name for now. +// String returns a human-readable string to display the name of this SQL node. func (t *DoltTable) String() string { return t.name } @@ -101,124 +149,6 @@ func (t *DoltTable) PartitionRows(ctx *sql.Context, _ sql.Partition) (sql.RowIte return newRowIterator(t, ctx) } -func (t *DoltTable) CreateIndex(ctx *sql.Context, indexName string, using sql.IndexUsing, constraint sql.IndexConstraint, columns []sql.IndexColumn, comment string) error { - if constraint != sql.IndexConstraint_None && constraint != sql.IndexConstraint_Unique { - return fmt.Errorf("not yet supported") - } - - if !doltdb.IsValidTableName(indexName) { - return fmt.Errorf("invalid index name `%s` as they must match the regular expression %s", indexName, doltdb.TableNameRegexStr) - } - - // get the real column names as CREATE INDEX columns are case-insensitive - var realColNames []string - allTableCols := t.sch.GetAllCols() - for _, indexCol := range columns { - tableCol, ok := allTableCols.GetByName(indexCol.Name) - if !ok { - tableCol, ok = allTableCols.GetByNameCaseInsensitive(indexCol.Name) - if !ok { - return fmt.Errorf("column `%s` does not exist for the table", indexCol.Name) - } - } - realColNames = append(realColNames, tableCol.Name) - } - - // create the index metadata, will error if index names are taken or an index with the same columns in the same order exists - _, err := t.sch.Indexes().AddIndexByColNames(indexName, realColNames, constraint == sql.IndexConstraint_Unique, comment) - if err != nil { - return err - } - - // update the table schema with the new index - newSchemaVal, err := encoding.MarshalSchemaAsNomsValue(ctx, t.table.ValueReadWriter(), t.sch) - if err != nil { - return err - } - tableRowData, err := t.table.GetRowData(ctx) - if err != nil { - return err - } - indexData, err := t.table.GetIndexData(ctx) - if err != nil { - return err - } - newTable, err := doltdb.NewTable(ctx, t.table.ValueReadWriter(), newSchemaVal, tableRowData, &indexData) - if err != nil { - return err - } - - // set the index row data and get a new root with the updated table - indexRowData, err := newTable.RebuildIndexRowData(ctx, indexName) - if err != nil { - return err - } - newTable, err = newTable.SetIndexRowData(ctx, indexName, indexRowData) - if err != nil { - return err - } - root, err := t.db.GetRoot(ctx) - if err != nil { - return err - } - newRoot, err := root.PutTable(ctx, t.name, newTable) - if err != nil { - return err - } - - return t.db.SetRoot(ctx, newRoot) -} - -func (t *DoltTable) DropIndex(ctx *sql.Context, indexName string) error { - // RemoveIndex returns an error if the index does not exist, no need to do twice - _, err := t.sch.Indexes().RemoveIndex(indexName) - if err != nil { - return err - } - newTable, err := t.table.UpdateSchema(ctx, t.sch) - if err != nil { - return err - } - - newTable, err = newTable.DeleteIndexRowData(ctx, indexName) - if err != nil { - return err - } - - root, err := t.db.GetRoot(ctx) - if err != nil { - return err - } - newRoot, err := root.PutTable(ctx, t.name, newTable) - if err != nil { - return err - } - return t.db.SetRoot(ctx, newRoot) -} - -func (t *DoltTable) RenameIndex(ctx *sql.Context, fromIndexName string, toIndexName string) error { - // RenameIndex will error if there is a name collision or an index does not exist - _, err := t.sch.Indexes().RenameIndex(fromIndexName, toIndexName) - if err != nil { - return err - } - newTable, err := t.table.UpdateSchema(ctx, t.sch) - if err != nil { - return err - } - - root, err := t.db.GetRoot(ctx) - if err != nil { - return err - } - newRoot, err := root.PutTable(ctx, t.name, newTable) - if err != nil { - return err - } - - return t.db.SetRoot(ctx, newRoot) -} - // WritableDoltTable allows updating, deleting, and inserting new rows. It implements sql.UpdatableTable and friends. type WritableDoltTable struct { DoltTable @@ -310,6 +240,7 @@ type AlterableDoltTable struct { } var _ sql.AlterableTable = (*AlterableDoltTable)(nil) +var _ sql.IndexAlterableTable = (*AlterableDoltTable)(nil) // AddColumn implements sql.AlterableTable func (t *AlterableDoltTable) AddColumn(ctx *sql.Context, column *sql.Column, order *sql.ColumnOrder) error { @@ -483,3 +414,124 @@ func (t *AlterableDoltTable) ModifyColumn(ctx *sql.Context, columnName string, c return t.db.SetRoot(ctx, newRoot) } + +// CreateIndex implements sql.IndexAlterableTable +func (t *AlterableDoltTable) CreateIndex(ctx *sql.Context, indexName string, using sql.IndexUsing, constraint sql.IndexConstraint, columns []sql.IndexColumn, comment string) error { + if constraint != sql.IndexConstraint_None && constraint != sql.IndexConstraint_Unique { + return fmt.Errorf("not yet supported") + } + + if !doltdb.IsValidTableName(indexName) { + return fmt.Errorf("invalid index name `%s` as they must match the regular expression %s", indexName, doltdb.TableNameRegexStr) + } + + // get the real column names as CREATE INDEX columns are case-insensitive + var realColNames []string + allTableCols := t.sch.GetAllCols() + for _, indexCol := range columns { + tableCol, ok := allTableCols.GetByName(indexCol.Name) + if !ok { + tableCol, ok = allTableCols.GetByNameCaseInsensitive(indexCol.Name) + if !ok { + return fmt.Errorf("column `%s` does not exist for the table", indexCol.Name) + } + } + realColNames = append(realColNames, tableCol.Name) + } + + // create the index metadata, will error if index names are taken or an index with the same columns in the same order exists + _, err := t.sch.Indexes().AddIndexByColNames(indexName, realColNames, constraint == sql.IndexConstraint_Unique, comment) + if err != nil { + return err + } + + // update the table schema with the new index + newSchemaVal, err := encoding.MarshalSchemaAsNomsValue(ctx, t.table.ValueReadWriter(), t.sch) + if err != nil { + return err + } + tableRowData, err := t.table.GetRowData(ctx) + if err != nil { + return err + } + indexData, err := t.table.GetIndexData(ctx) + if err != nil { + return err + } + newTable, err := doltdb.NewTable(ctx, t.table.ValueReadWriter(), newSchemaVal, tableRowData, &indexData) + if err != nil { + return err + } + + // set the index row data and get a new root with the updated table + indexRowData, err := newTable.RebuildIndexRowData(ctx, indexName) + if err != nil { + return err + } + newTable, err = newTable.SetIndexRowData(ctx, indexName, indexRowData) + if err != nil { + return err + } + root, err := t.db.GetRoot(ctx) + if err != nil { + return err + } + newRoot, err := root.PutTable(ctx, t.name, newTable) + if err != nil { + return err + } + + return t.db.SetRoot(ctx, newRoot) +} + +// DropIndex implements sql.IndexAlterableTable +func (t *AlterableDoltTable) DropIndex(ctx *sql.Context, indexName string) error { + // RemoveIndex returns an error if the index does not exist, no need to do twice + _, err := t.sch.Indexes().RemoveIndex(indexName) + if err != nil { + return err + } + newTable, err := t.table.UpdateSchema(ctx, t.sch) + if err != nil { + return err + } + + newTable, err = newTable.DeleteIndexRowData(ctx, indexName) + if err != nil { + return err + } + + root, err := t.db.GetRoot(ctx) + if err != nil { + return err + } + newRoot, err := root.PutTable(ctx, t.name, newTable) + if err != nil { + return err + } + return t.db.SetRoot(ctx, newRoot) +} + +// RenameIndex implements sql.IndexAlterableTable +func (t *AlterableDoltTable) RenameIndex(ctx *sql.Context, fromIndexName string, toIndexName string) error { + // RenameIndex will error if there is a name collision or an index does not exist + _, err := t.sch.Indexes().RenameIndex(fromIndexName, toIndexName) + if err != nil { + return err + } + newTable, err := t.table.UpdateSchema(ctx, t.sch) + if err != nil { + return err + } + + root, err := t.db.GetRoot(ctx) + if err != nil { + return err + } + newRoot, err := root.PutTable(ctx, t.name, newTable) + if err != nil { + return err + } + + return t.db.SetRoot(ctx, newRoot) +} diff --git a/go/libraries/doltcore/sqle/testutil.go b/go/libraries/doltcore/sqle/testutil.go index 16f58c381e..b2dd4fabde 100644 --- a/go/libraries/doltcore/sqle/testutil.go +++ b/go/libraries/doltcore/sqle/testutil.go @@ -112,13 +112,6 @@ func NewTestEngine(ctx context.Context, db Database, root *doltdb.RootValue) (*s return nil, nil, err } - sqlCtx.RegisterIndexDriver(NewDoltIndexDriver(db)) - err = sqlCtx.LoadIndexes(sqlCtx, engine.Catalog.AllDatabases()) - - if err != nil { - return nil, nil, err - } - err = RegisterSchemaFragments(sqlCtx, db, root) if err != nil { From 28e5b3be97588ef705932e74f7ebb7a71b0482a3 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 22 May 2020 14:36:46 -0700 Subject: [PATCH 4/7] Filled in some comments and did some renaming Signed-off-by: Zach Musgrave --- go/libraries/doltcore/sqle/dolt_index.go | 29 ++++++++++++++++------ go/libraries/doltcore/sqle/index_lookup.go | 4 +-- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/go/libraries/doltcore/sqle/dolt_index.go b/go/libraries/doltcore/sqle/dolt_index.go index b89d00bb57..3aab992068 100644 --- a/go/libraries/doltcore/sqle/dolt_index.go +++ b/go/libraries/doltcore/sqle/dolt_index.go @@ -53,24 +53,27 @@ var alwaysContinueRangeCheck noms.InRangeCheck = func(tuple types.Tuple) (bool, return true, nil } +// AscendGreaterOrEqual implements sql.AscendIndex func (di *doltIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { tpl, err := di.keysToTuple(keys, false) if err != nil { return nil, err } readRange := &noms.ReadRange{Start: tpl, Inclusive: true, Reverse: false, Check: alwaysContinueRangeCheck} - return di.rangeToIter(readRange) + return di.rangeToIndexLookup(readRange) } +// AscendLessThan implements sql.AscendIndex func (di *doltIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { tpl, err := di.keysToTuple(keys, false) if err != nil { return nil, err } readRange := &noms.ReadRange{Start: tpl, Inclusive: false, Reverse: true, Check: alwaysContinueRangeCheck} - return di.rangeToIter(readRange) + return di.rangeToIndexLookup(readRange) } +// AscendRange implements sql.AscendIndex // TODO: rename this from AscendRange to BetweenRange or something func (di *doltIndex) AscendRange(greaterOrEqual, lessThanOrEqual []interface{}) (sql.IndexLookup, error) { greaterTpl, err := di.keysToTuple(greaterOrEqual, false) @@ -85,40 +88,46 @@ func (di *doltIndex) AscendRange(greaterOrEqual, lessThanOrEqual []interface{}) readRange := &noms.ReadRange{Start: greaterTpl, Inclusive: true, Reverse: false, Check: func(tuple types.Tuple) (bool, error) { return tuple.Less(nbf, lessTpl) }} - return di.rangeToIter(readRange) + return di.rangeToIndexLookup(readRange) } +// DescendGreater implements sql.DescendIndex func (di *doltIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { tpl, err := di.keysToTuple(keys, true) if err != nil { return nil, err } readRange := &noms.ReadRange{Start: tpl, Inclusive: true, Reverse: false, Check: alwaysContinueRangeCheck} - return di.rangeToIter(readRange) + return di.rangeToIndexLookup(readRange) } +// DescendLessOrEqual implements sql.DescendIndex func (di *doltIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { tpl, err := di.keysToTuple(keys, true) if err != nil { return nil, err } readRange := &noms.ReadRange{Start: tpl, Inclusive: true, Reverse: true, Check: alwaysContinueRangeCheck} - return di.rangeToIter(readRange) + return di.rangeToIndexLookup(readRange) } +// DescendRange implements sql.DescendIndex // TODO: fix go-mysql-server to remove this duplicate function func (di *doltIndex) DescendRange(lessOrEqual, greaterOrEqual []interface{}) (sql.IndexLookup, error) { return di.AscendRange(greaterOrEqual, lessOrEqual) } +// Database implement sql.Index func (di *doltIndex) Database() string { return di.db.name } +// DoltDatabase returns the dolt database that created this index. func (di *doltIndex) DoltDatabase() Database { return di.db } +// Expressions implements sql.Index func (di *doltIndex) Expressions() []string { strs := make([]string, len(di.cols)) for i, col := range di.cols { @@ -127,6 +136,7 @@ func (di *doltIndex) Expressions() []string { return strs } +// Get implements sql.Index func (di *doltIndex) Get(keys ...interface{}) (sql.IndexLookup, error) { tpl, err := di.keysToTuple(keys, false) if err != nil { @@ -135,26 +145,31 @@ func (di *doltIndex) Get(keys ...interface{}) (sql.IndexLookup, error) { readRange := &noms.ReadRange{Start: tpl, Inclusive: true, Reverse: false, Check: func(tuple types.Tuple) (bool, error) { return tuple.StartsWith(tpl), nil }} - return di.rangeToIter(readRange) + return di.rangeToIndexLookup(readRange) } +// Has implements sql.Index func (*doltIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) { // appears to be unused for the moment panic("implement me") } +// ID implements sql.Index func (di *doltIndex) ID() string { return di.id } +// Schema returns the dolt schema of this index. func (di *doltIndex) Schema() schema.Schema { return di.tableSch } +// Table implements sql.Index func (di *doltIndex) Table() string { return di.tableName } +// TableData returns the map of table data for this index (the map of the index table, not the target table) func (di *doltIndex) TableData() types.Map { return di.tableData } @@ -180,7 +195,7 @@ func (di *doltIndex) keysToTuple(keys []interface{}, appendMaxValue bool) (types return types.NewTuple(nbf, vals...) } -func (di *doltIndex) rangeToIter(readRange *noms.ReadRange) (sql.IndexLookup, error) { +func (di *doltIndex) rangeToIndexLookup(readRange *noms.ReadRange) (sql.IndexLookup, error) { var mapIter table.TableReadCloser = noms.NewNomsRangeReader(di.indexSch, di.indexRowData, []*noms.ReadRange{readRange}) return &doltIndexLookup{ di, diff --git a/go/libraries/doltcore/sqle/index_lookup.go b/go/libraries/doltcore/sqle/index_lookup.go index 2b8ecdfdbf..ce0b02bca8 100644 --- a/go/libraries/doltcore/sqle/index_lookup.go +++ b/go/libraries/doltcore/sqle/index_lookup.go @@ -33,8 +33,8 @@ type doltIndexLookup struct { } func (il *doltIndexLookup) String() string { - // TODO: fix - return fmt.Sprintf("%s:%s", il.idx.ID(), "") + // TODO: this could be expanded with additional info (like the expression used to create the index lookup) + return fmt.Sprintf("doltIndexLookup:%s", il.idx.ID()) } var _ sql.IndexLookup = (*doltIndexLookup)(nil) From 7db30f9d340e1b96422cfa7437d2d119ed790efd Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 22 May 2020 14:41:59 -0700 Subject: [PATCH 5/7] Updated to latest go-mysql-server Signed-off-by: Zach Musgrave --- go/go.mod | 5 +---- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go/go.mod b/go/go.mod index e06bf10bd6..7f75d72526 100644 --- a/go/go.mod +++ b/go/go.mod @@ -44,7 +44,7 @@ require ( github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/kr/pretty v0.2.0 // indirect github.com/liquidata-inc/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20200320155049-a8e482faeffd - github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200519213537-130e749c2a7d + github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200522204835-54fb48963b57 github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0 github.com/liquidata-inc/mmap-go v1.0.3 github.com/liquidata-inc/sqllogictest/go v0.0.0-20200320151923-b11801f10e15 @@ -52,7 +52,6 @@ require ( github.com/mattn/go-runewidth v0.0.9 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b github.com/miekg/dns v1.1.27 // indirect - github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.6.0 // indirect github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.4.0 @@ -90,8 +89,6 @@ require ( replace github.com/liquidata-inc/dolt/go/gen/proto/dolt/services/eventsapi => ./gen/proto/dolt/services/eventsapi -replace github.com/liquidata-inc/go-mysql-server => ../../go-mysql-server - replace vitess.io/vitess => github.com/liquidata-inc/vitess v0.0.0-20200430040751-192bb76ecd8b go 1.13 diff --git a/go/go.sum b/go/go.sum index b3717707a3..e39bb83e7b 100644 --- a/go/go.sum +++ b/go/go.sum @@ -392,8 +392,8 @@ github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200519213537-130e749c2a7d h1:j2x7fCcQcibMFdXzPkUG3+0aWUvqL6+9W7Iga555A14= -github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200519213537-130e749c2a7d/go.mod h1:Qo0l83LdX5Z77p0tTLyJTrttZywFm0S+RYo6Shi97tw= +github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200522204835-54fb48963b57 h1:jtx0pubYH4r7G0mK7SioB/HYwpNqL587nPnbLur+fPE= +github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200522204835-54fb48963b57/go.mod h1:Qo0l83LdX5Z77p0tTLyJTrttZywFm0S+RYo6Shi97tw= github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0 h1:phMgajKClMUiIr+hF2LGt8KRuUa2Vd2GI1sNgHgSXoU= github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0/go.mod h1:YC1rI9k5gx8D02ljlbxDfZe80s/iq8bGvaaQsvR+qxs= github.com/liquidata-inc/mmap-go v1.0.3 h1:2LndAeAtup9rpvUmu4wZSYCsjCQ0Zpc+NqE+6+PnT7g= From 7a44bca8b46ff76af2361487389a8fd3e74a3ee0 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 26 May 2020 11:01:30 -0700 Subject: [PATCH 6/7] PR feedback Signed-off-by: Zach Musgrave --- go/libraries/doltcore/sqle/dolt_index.go | 2 +- go/libraries/doltcore/sqle/indexed_dolt_table.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/dolt_index.go b/go/libraries/doltcore/sqle/dolt_index.go index 3aab992068..3129d0e35f 100644 --- a/go/libraries/doltcore/sqle/dolt_index.go +++ b/go/libraries/doltcore/sqle/dolt_index.go @@ -169,7 +169,7 @@ func (di *doltIndex) Table() string { return di.tableName } -// TableData returns the map of table data for this index (the map of the index table, not the target table) +// TableData returns the map of table data for this index (the map of the target table, not the index storage table) func (di *doltIndex) TableData() types.Map { return di.tableData } diff --git a/go/libraries/doltcore/sqle/indexed_dolt_table.go b/go/libraries/doltcore/sqle/indexed_dolt_table.go index 4a2960fb81..c0636e3367 100644 --- a/go/libraries/doltcore/sqle/indexed_dolt_table.go +++ b/go/libraries/doltcore/sqle/indexed_dolt_table.go @@ -27,8 +27,7 @@ type IndexedDoltTable struct { var _ sql.IndexedTable = (*IndexedDoltTable)(nil) func (idt *IndexedDoltTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) { - // TODO: move index driver implementation here - panic("implement me") + return idt.table.GetIndexes(ctx) } func (idt *IndexedDoltTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table { From 330fb06d2b2d59dbad2d8e1297fb94ecf08b28dc Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 26 May 2020 11:15:06 -0700 Subject: [PATCH 7/7] Formatting Signed-off-by: Zach Musgrave --- go/libraries/doltcore/sqle/index_lookup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/libraries/doltcore/sqle/index_lookup.go b/go/libraries/doltcore/sqle/index_lookup.go index ce0b02bca8..653d6a1bcc 100644 --- a/go/libraries/doltcore/sqle/index_lookup.go +++ b/go/libraries/doltcore/sqle/index_lookup.go @@ -16,6 +16,7 @@ package sqle import ( "fmt" + "github.com/liquidata-inc/go-mysql-server/sql" "github.com/liquidata-inc/dolt/go/libraries/doltcore/row"