Almost working table / row iterators

go/cmd/dolt/commands/sqle/sqle_test.go

@@ -0,0 +1,484 @@
+package sqle
+
+import (
+	"context"
+	"github.com/attic-labs/noms/go/types"
+	"github.com/google/uuid"
+	"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/dtestutils"
+	"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/env"
+	"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/row"
+	"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table"
+	"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/typed/noms"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+
+//var UUIDS = []uuid.UUID{
+//	uuid.Must(uuid.Parse("00000000-0000-0000-0000-000000000000")),
+//	uuid.Must(uuid.Parse("00000000-0000-0000-0000-000000000001")),
+//	uuid.Must(uuid.Parse("00000000-0000-0000-0000-000000000002"))}
+//var Names = []string{"Bill Billerson", "John Johnson", "Rob Robertson"}
+//var Ages = []uint64{32, 25, 21}
+//var Titles = []string{"Senior Dufus", "Dufus", ""}
+//var MaritalStatus = []bool{true, false, false}
+
+var tableName = "people"
+
+// Smoke tests, values are printed to console
+func TestSqlSelect(t *testing.T) {
+	tests := []struct {
+		query       string
+		expectedRes int
+	}{
+//		{"select * from doesnt_exist where age = 32", 1},
+		{"select * from people", 0},
+		{"select * from people where age = 32", 0},
+		{"select * from people where title = 'Senior Dufus'", 0},
+		{"select * from people where name = 'Bill Billerson'", 0},
+		{"select * from people where name = 'John Johnson'", 0},
+		{"select * from people where age = 25", 0},
+		{"select * from people where 25 = age", 0},
+		{"select * from people where is_married = false", 0},
+		{"select * from people where age < 30", 0},
+		{"select * from people where age > 24", 0},
+		{"select * from people where age >= 25", 0},
+		{"select * from people where name <= 'John Johnson'", 0},
+		{"select * from people where name <> 'John Johnson'", 0},
+		{"select age, is_married from people where name <> 'John Johnson'", 0},
+		{"select age, is_married from people where name <> 'John Johnson' limit 1", 0},
+	}
+
+	for _, test := range tests {
+		t.Run(test.query, func(t *testing.T) {
+			dEnv := createEnvWithSeedData(t)
+
+			args := []string{"-q", test.query}
+
+			commandStr := "dolt sqle"
+			result := Sql(commandStr, args, dEnv)
+			assert.Equal(t, test.expectedRes, result)
+		})
+	}
+}
+
+// Smoke tests, values are printed to console
+func TestSqlShow(t *testing.T) {
+	tests := []struct {
+		query       string
+		expectedRes int
+	}{
+		{"show tables", 0},
+		{"show create table people", 0},
+		{"show all tables", 1},
+	}
+
+	for _, test := range tests {
+		t.Run(test.query, func(t *testing.T) {
+			dEnv := createEnvWithSeedData(t)
+
+			args := []string{"-q", test.query}
+
+			commandStr := "dolt sqle"
+			result := Sql(commandStr, args, dEnv)
+			assert.Equal(t, test.expectedRes, result)
+		})
+	}
+}
+
+func TestBadInput(t *testing.T) {
+	tests := []struct {
+		name        string
+		args        []string
+		expectedRes int
+	}{
+		{"no query", []string{"-q", ""}, 1},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			dEnv := createEnvWithSeedData(t)
+
+			commandStr := "dolt sqle"
+			result := Sql(commandStr, test.args, dEnv)
+			assert.Equal(t, test.expectedRes, result)
+		})
+	}
+}
+
+// Tests of the create table SQL command, mostly a smoke test for errors in the command line handler. Most tests of
+// create table SQL command are in the sql package.
+func TestCreateTable(t *testing.T) {
+	tests := []struct {
+		query       string
+		expectedRes int
+	}{
+		{"create table people (id int)", 1}, // no primary key
+		{"create table", 1},                 // bad syntax
+		{"create table (id int ", 1},        // bad syntax
+		{"create table people (id int primary key)", 0},
+		{"create table people (id int primary key, age int)", 0},
+		{"create table people (id int primary key, age int, first varchar(80), is_married bit)", 0},
+		{"create table people (`id` int, `age` int, `first` varchar(80), `last` varchar(80), `title` varchar(80), `is_married` bit, primary key (`id`, `age`))", 0},
+	}
+
+	for _, test := range tests {
+		t.Run(test.query, func(t *testing.T) {
+			dEnv := dtestutils.CreateTestEnv()
+			working, err := dEnv.WorkingRoot(context.Background())
+			assert.Nil(t, err, "Unexpected error")
+			assert.False(t, working.HasTable(context.Background(), tableName), "table exists before creating it")
+
+			args := []string{"-q", test.query}
+			commandStr := "dolt sqle"
+			result := Sql(commandStr, args, dEnv)
+			assert.Equal(t, test.expectedRes, result)
+
+			working, err = dEnv.WorkingRoot(context.Background())
+			assert.Nil(t, err, "Unexpected error")
+			if test.expectedRes == 0 {
+				assert.True(t, working.HasTable(context.Background(), tableName), "table doesn't exist after creating it")
+			} else {
+				assert.False(t, working.HasTable(context.Background(), tableName), "table shouldn't exist after error")
+			}
+		})
+	}
+}
+
+// Tests of the create table SQL command, mostly a smoke test for errors in the command line handler. Most tests of
+// create table SQL command are in the sql package.
+func TestShowTables(t *testing.T) {
+	tests := []struct {
+		query       string
+		expectedRes int
+	}{
+		{"show ", 1},                 // bad syntax
+		{"show table", 1},        // bad syntax
+		{"show tables", 0},
+		{"show create table people", 0},
+		{"show create table dne", 1},
+	}
+
+	for _, test := range tests {
+		t.Run(test.query, func(t *testing.T) {
+			dEnv := createEnvWithSeedData(t)
+
+			args := []string{"-q", test.query}
+			commandStr := "dolt sqle"
+			result := Sql(commandStr, args, dEnv)
+			assert.Equal(t, test.expectedRes, result)
+		})
+	}
+}
+
+
+// Tests of the alter table SQL command, mostly a smoke test for errors in the command line handler. Most tests of
+// create table SQL command are in the sql package.
+func TestAlterTable(t *testing.T) {
+	tests := []struct {
+		query       string
+		expectedRes int
+	}{
+		{"alter table", 1},                 // bad syntax
+		{"alter table people rename", 1},        // bad syntax
+		{"alter table dne rename id to newId", 1},        // unknown column
+		{"alter table people rename id to newId", 0}, // no primary key
+		{"alter table people rename to newPeople", 0},
+		{"rename table people to newPeople", 0},
+		{"alter table people add column (newCol int not null default 10)", 0},
+		{"alter table people drop column title", 0},
+	}
+
+	for _, test := range tests {
+		t.Run(test.query, func(t *testing.T) {
+			dEnv := createEnvWithSeedData(t)
+
+			args := []string{"-q", test.query}
+			commandStr := "dolt sqle"
+			result := Sql(commandStr, args, dEnv)
+			assert.Equal(t, test.expectedRes, result)
+		})
+	}
+}
+
+// Tests of the drop table SQL command, mostly a smoke test for errors in the command line handler. Most tests of
+// create table SQL command are in the sql package.
+func TestDropTable(t *testing.T) {
+	tests := []struct {
+		query       string
+		expectedRes int
+	}{
+		{"drop table", 1},
+		{"drop table people", 0},
+		{"drop table dne", 1},
+		{"drop table if exists dne", 0},
+	}
+
+	for _, test := range tests {
+		t.Run(test.query, func(t *testing.T) {
+			dEnv := createEnvWithSeedData(t)
+
+			args := []string{"-q", test.query}
+			commandStr := "dolt sqle"
+			result := Sql(commandStr, args, dEnv)
+			assert.Equal(t, test.expectedRes, result)
+		})
+	}
+}
+
+// Tests of the insert SQL command, mostly a smoke test for errors in the command line handler. Most tests of
+// insert SQL command are in the sql package.
+func TestInsert(t *testing.T) {
+	tests := []struct {
+		name        string
+		query       string
+		expectedRes int
+		expectedIds []uuid.UUID
+	}{
+		{
+			name:        "no primary key",
+			query:       "insert into people (title) values ('hello')",
+			expectedRes: 1,
+		},
+		{
+			name:  "bad syntax",
+			query: "insert into table", expectedRes: 1,
+		},
+		{
+			name:  "bad syntax",
+			query: "insert into people (id) values", expectedRes: 1,
+		},
+		{
+			name:  "table doesn't exist",
+			query: "insert into dne (id) values (00000000-0000-0000-0000-000000000005)", expectedRes: 1,
+		},
+		{
+			name: "insert one row",
+			query: `insert into people (id, name, age, is_married) values 
+				('00000000-0000-0000-0000-000000000005', 'Frank Frankerson', 10, false)`,
+			expectedIds: []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000005")},
+		},
+		{
+			name: "insert one row all columns",
+			query: `insert into people (id, name, age, is_married, title) values 
+				('00000000-0000-0000-0000-000000000005', 'Frank Frankerson', 10, false, 'Goon')`,
+			expectedIds: []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000005")},
+		},
+		{
+			name: "insert two rows all columns",
+			query: `insert into people (id, name, age, is_married, title) values 
+				('00000000-0000-0000-0000-000000000005', 'Frank Frankerson', 10, false, 'Goon'),
+				('00000000-0000-0000-0000-000000000006', 'Kobe Buffalomeat', 30, false, 'Linebacker')`,
+			expectedIds: []uuid.UUID{
+				uuid.MustParse("00000000-0000-0000-0000-000000000005"),
+				uuid.MustParse("00000000-0000-0000-0000-000000000006"),
+			},
+		},
+		{
+			name: "missing required column",
+			query: `insert into people (id, name, age) values 
+				('00000000-0000-0000-0000-000000000005', 'Frank Frankerson', 10)`,
+			expectedRes: 1,
+		},
+		{
+			name: "existing primary key",
+			query: `insert into people (id, name, age, is_married, title) values 
+				('00000000-0000-0000-0000-000000000000', 'Frank Frankerson', 10, false, 'Goon')`,
+			expectedRes: 1,
+		},
+		{
+			name: "insert ignore",
+			query: `insert ignore into people (id, name, age, is_married, title) values 
+				('00000000-0000-0000-0000-000000000000', 'Frank Frankerson', 10, false, 'Goon')`,
+			expectedIds: []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000000")},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.query, func(t *testing.T) {
+			dEnv := createEnvWithSeedData(t)
+
+			args := []string{"-q", test.query}
+
+			commandStr := "dolt sqle"
+			result := Sql(commandStr, args, dEnv)
+			assert.Equal(t, test.expectedRes, result)
+
+			if result == 0 {
+				root, err := dEnv.WorkingRoot(context.Background())
+				assert.Nil(t, err)
+
+				// Assert that all expected IDs exist after the insert
+				for _, expectedid := range test.expectedIds {
+					table, _ := root.GetTable(context.Background(), tableName)
+					taggedVals := row.TaggedValues{dtestutils.IdTag: types.UUID(expectedid)}
+					key := taggedVals.NomsTupleForTags([]uint64{dtestutils.IdTag}, true)
+					_, ok := table.GetRow(context.Background(), key.Value(context.Background()).(types.Tuple), dtestutils.TypedSchema)
+					assert.True(t, ok, "expected id not found")
+				}
+			}
+		})
+	}
+}
+
+// Tests of the update SQL command, mostly a smoke test for errors in the command line handler. Most tests of
+// update SQL command are in the sql package.
+func TestUpdate(t *testing.T) {
+	tests := []struct {
+		name         string
+		query        string
+		expectedRes  int
+		expectedIds  []uuid.UUID
+		expectedAges []uint
+	}{
+		{
+			name:  "bad syntax",
+			query: "update table", expectedRes: 1,
+		},
+		{
+			name:  "bad syntax",
+			query: "update people set id", expectedRes: 1,
+		},
+		{
+			name:  "table doesn't exist",
+			query: "update dne set id = '00000000-0000-0000-0000-000000000005'", expectedRes: 1,
+		},
+		{
+			name:         "update one row",
+			query:        `update people set age = 1 where id = '00000000-0000-0000-0000-000000000002'`,
+			expectedIds:  []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000002")},
+			expectedAges: []uint{1},
+		},
+		{
+			name:  "insert two rows, two columns",
+			query: `update people set age = 1, is_married = true where age > 21`,
+			expectedIds: []uuid.UUID{
+				uuid.MustParse("00000000-0000-0000-0000-000000000000"),
+				uuid.MustParse("00000000-0000-0000-0000-000000000001"),
+			},
+			expectedAges: []uint{1, 1},
+		},
+		{
+			name:        "null constraint violation",
+			query:       `update people set name = null where id ='00000000-0000-0000-0000-000000000000'`,
+			expectedRes: 1,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.query, func(t *testing.T) {
+			ctx := context.Background()
+			dEnv := createEnvWithSeedData(t)
+
+			args := []string{"-q", test.query}
+
+			commandStr := "dolt sqle"
+			result := Sql(commandStr, args, dEnv)
+			assert.Equal(t, test.expectedRes, result)
+
+			if result == 0 {
+				root, err := dEnv.WorkingRoot(context.Background())
+				assert.Nil(t, err)
+
+				// Assert that all rows have been updated
+				for i, expectedid := range test.expectedIds {
+					table, _ := root.GetTable(context.Background(), tableName)
+					taggedVals := row.TaggedValues{dtestutils.IdTag: types.UUID(expectedid)}
+					key := taggedVals.NomsTupleForTags([]uint64{dtestutils.IdTag}, true)
+					row, ok := table.GetRow(ctx, key.Value(ctx).(types.Tuple), dtestutils.TypedSchema)
+					assert.True(t, ok, "expected id not found")
+					ageVal, _ := row.GetColVal(dtestutils.AgeTag)
+					assert.Equal(t, test.expectedAges[i], uint(ageVal.(types.Uint)))
+				}
+			}
+		})
+	}
+}
+
+// Tests of the delete SQL command, mostly a smoke test for errors in the command line handler. Most tests of
+// delete SQL command are in the sql package.
+func TestDelete(t *testing.T) {
+	tests := []struct {
+		name        string
+		query       string
+		expectedRes int
+		deletedIds  []uuid.UUID
+	}{
+		{
+			name:  "bad syntax",
+			query: "delete table", expectedRes: 1,
+		},
+		{
+			name:  "bad syntax",
+			query: "delete from people where", expectedRes: 1,
+		},
+		{
+			name:  "table doesn't exist",
+			query: "delete from dne", expectedRes: 1,
+		},
+		{
+			name:       "delete one row",
+			query:      `delete from people where id = '00000000-0000-0000-0000-000000000002'`,
+			deletedIds: []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000002")},
+		},
+		{
+			name:  "delete two rows",
+			query: `delete from people where age > 21`,
+			deletedIds: []uuid.UUID{
+				uuid.MustParse("00000000-0000-0000-0000-000000000000"),
+				uuid.MustParse("00000000-0000-0000-0000-000000000001"),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.query, func(t *testing.T) {
+			dEnv := createEnvWithSeedData(t)
+			ctx := context.Background()
+
+			args := []string{"-q", test.query}
+
+			commandStr := "dolt sqle"
+			result := Sql(commandStr, args, dEnv)
+			assert.Equal(t, test.expectedRes, result)
+
+			if result == 0 {
+				root, err := dEnv.WorkingRoot(context.Background())
+				assert.Nil(t, err)
+
+				// Assert that all rows have been deleted
+				for _, expectedid := range test.deletedIds {
+					table, _ := root.GetTable(context.Background(), tableName)
+					taggedVals := row.TaggedValues{dtestutils.IdTag: types.UUID(expectedid)}
+					key := taggedVals.NomsTupleForTags([]uint64{dtestutils.IdTag}, true)
+					_, ok := table.GetRow(ctx, key.Value(ctx).(types.Tuple), dtestutils.TypedSchema)
+					assert.False(t, ok, "row not deleted")
+				}
+			}
+		})
+	}
+}
+
+func createEnvWithSeedData(t *testing.T) *env.DoltEnv {
+	dEnv := dtestutils.CreateTestEnv()
+	imt, sch := dtestutils.CreateTestDataTable(true)
+
+	rd := table.NewInMemTableReader(imt)
+	wr := noms.NewNomsMapCreator(context.Background(), dEnv.DoltDB.ValueReadWriter(), sch)
+
+	_, _, err := table.PipeRows(context.Background(), rd, wr, false)
+	rd.Close(context.Background())
+	wr.Close(context.Background())
+
+	if err != nil {
+		t.Error("Failed to seed initial data", err)
+	}
+
+	err = dEnv.PutTableToWorking(context.Background(), *wr.GetMap(), wr.GetSchema(), tableName)
+
+	if err != nil {
+		t.Error("Unable to put initial value of table in in mem noms db", err)
+	}
+
+	return dEnv
+}
+

go/libraries/doltcore/sqle/tables.go

@@ -2,9 +2,13 @@ package sqle
 
 import (
 	"context"
+	"github.com/attic-labs/noms/go/types"
 	"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/doltdb"
 	"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/env"
+	"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/row"
+	"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/schema"
 	"github.com/src-d/go-mysql-server/sql"
+	"io"
 )
 
 type Database struct {
@@ -13,8 +17,9 @@ type Database struct {
 }
 
 type DoltTable struct {
-	name string
+	name  string
 	table *doltdb.Table
+	sch   schema.Schema
 }
 
 func (t *DoltTable) Name() string {
@@ -30,12 +35,90 @@ func (t *DoltTable) Schema() sql.Schema {
 	return doltSchemaToSqlSchema(t.name, schema)
 }
 
-func (*DoltTable) Partitions(*sql.Context) (sql.PartitionIter, error) {
-	panic("implement me")
+type doltTablePartitionIter struct {
+	table *DoltTable
+	i int
 }
 
-func (*DoltTable) PartitionRows(*sql.Context, sql.Partition) (sql.RowIter, error) {
-	panic("implement me")
+func (itr *doltTablePartitionIter) Close() error {
+	return nil
+}
+
+func (itr *doltTablePartitionIter) Next() (sql.Partition, error) {
+	if itr.i > 0 {
+		return nil, io.EOF
+	}
+	itr.i++
+
+	return &doltTablePartition{itr.table}, nil
+}
+
+type doltTablePartition struct {
+	table *DoltTable
+}
+
+func (p doltTablePartition) Key() []byte {
+	return []byte(p.table.name)
+}
+
+// Returns the partitions for this table. We return a single partition, but could potentially get more performance by
+// returning multiple.
+func (t *DoltTable) Partitions(*sql.Context) (sql.PartitionIter, error) {
+	return &doltTablePartitionIter{table: t}, nil
+}
+
+// Returns the table rows for the partition given.
+func (t *DoltTable) PartitionRows(ctx *sql.Context, p sql.Partition) (sql.RowIter, error) {
+	return newRowIterator(t, ctx), nil
+}
+
+type doltTableRowIter struct {
+	table *DoltTable
+	rowData types.Map
+	ctx *sql.Context
+	nomsIter types.MapIterator
+}
+
+func newRowIterator(tbl *DoltTable, ctx *sql.Context) *doltTableRowIter {
+	rowData := tbl.table.GetRowData(ctx.Context)
+	mapIter := rowData.Iterator(ctx.Context)
+	return &doltTableRowIter{table: tbl, rowData: rowData, ctx: ctx, nomsIter: mapIter}
+}
+
+func (itr *doltTableRowIter) Next() (sql.Row, error) {
+	key, val := itr.nomsIter.Next(itr.ctx.Context)
+	if key == nil && val == nil {
+		return nil, io.EOF
+	}
+
+	doltRow := row.FromNoms(itr.table.sch, key.(types.Tuple), val.(types.Tuple))
+	return doltRowToSqlRow(doltRow, itr.table.sch), nil
+}
+
+func doltRowToSqlRow(doltRow row.Row, sch schema.Schema) sql.Row {
+	colVals := make(sql.Row, sch.GetAllCols().Size())
+
+	i := 0
+	sch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool) {
+		value, _:= doltRow.GetColVal(tag)
+		colVals[i] = doltColValToSqlColVal(value)
+		i++
+		return false
+	})
+
+	return sql.NewRow(colVals)
+}
+
+func doltColValToSqlColVal(val types.Value) interface{} {
+	if types.IsNull(val) {
+		return nil
+	}
+
+	return nomsValToSqlVal(val)
+}
+
+func (itr *doltTableRowIter) Close() error {
+	return nil
 }
 
 func NewDatabase(name string, dEnv *env.DoltEnv) sql.Database {
@@ -63,7 +146,7 @@ func (db *Database) Tables() map[string]sql.Table {
 		if !ok {
 			panic("Error loading table " + name)
 		}
-		tables[name] = &DoltTable{name: name, table: table}
+		tables[name] = &DoltTable{name: name, table: table, sch: table.GetSchema(context.TODO())}
 	}
 
 	return tables

go/libraries/doltcore/sqle/types.go

@@ -15,7 +15,8 @@ func nomsTypeToSqlType(kind types.NomsKind) sql.Type {
 	case types.StringKind:
 		return sql.Text
 	case types.UUIDKind:
-		panic("TODO")
+		// TODO: make an actual uuid
+		return sql.Text
 	case types.IntKind:
 		return sql.Int64
 	case types.UintKind:
@@ -24,3 +25,46 @@ func nomsTypeToSqlType(kind types.NomsKind) sql.Type {
 		panic(fmt.Sprintf("Unexpected kind %v", kind))
 	}
 }
+
+func nomsValToSqlVal(val types.Value) interface{} {
+	switch val.Kind() {
+	case types.BoolKind:
+		return convertBool(val.(types.Bool))
+	case types.FloatKind:
+		return convertFloat(val.(types.Float))
+	case types.StringKind:
+		return convertString(val.(types.String))
+	case types.UUIDKind:
+		return convertUUID(val.(types.UUID))
+	case types.IntKind:
+		return convertInt(val.(types.Int))
+	case types.UintKind:
+		return convertUint(val.(types.Uint))
+	default:
+		panic(fmt.Sprintf("Unexpected kind %v", val.Kind()))
+	}
+}
+
+func convertUUID(u types.UUID) interface{} {
+	return u.String()
+}
+
+func convertUint(i types.Uint) interface{} {
+	return uint64(i)
+}
+
+func convertInt(i types.Int) interface{} {
+	return int64(i)
+}
+
+func convertString(i types.String) interface{} {
+	return string(i)
+}
+
+func convertFloat(f types.Float) interface{} {
+	return float64(f)
+}
+
+func convertBool(b types.Bool) interface{} {
+	return bool(b)
+}