Merge pull request #3493 from dolthub/andy/prolly-enum-set-bit-encodings

[no-release-notes] Prolly Encodings for `Enum`, `Set` and `Bit`

go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go

@@ -76,33 +76,20 @@ func TestSingleQuery(t *testing.T) {
 
 // Convenience test for debugging a single query. Unskip and set to the desired query.
 func TestSingleScript(t *testing.T) {
-	t.Skip()
-
 	var scripts = []queries.ScriptTest{
 		{
-			Name: "Drop and add primary key on two branches converges to same schema",
+			Name: "Create table with TIME type",
 			SetUpScript: []string{
-				"create table t1 (i int);",
-				"call dolt_commit('-am', 't1 table')",
-				"call dolt_checkout('-b', 'b1')",
-				"alter table t1 add primary key(i)",
-				"alter table t1 drop primary key",
-				"alter table t1 add primary key(i)",
-				"alter table t1 drop primary key",
-				"alter table t1 add primary key(i)",
-				"call dolt_commit('-am', 'b1 primary key changes')",
-				"call dolt_checkout('main')",
-				"alter table t1 add primary key(i)",
-				"call dolt_commit('-am', 'main primary key change')",
+				"create table my_types (pk int primary key, c0 time);",
 			},
 			Assertions: []queries.ScriptTestAssertion{
 				{
-					Query:    "call dolt_merge('b1')",
-					Expected: []sql.Row{{1}},
+					Query:    "INSERT INTO my_types VALUES (1, '11:22:33.444444');",
+					Expected: []sql.Row{{sql.OkResult{RowsAffected: 1, InsertID: 0}}},
 				},
 				{
-					Query:    "select count(*) from dolt_conflicts",
-					Expected: []sql.Row{{0}},
+					Query:    "UPDATE my_types SET c0='11:22' WHERE pk=1;",
+					Expected: []sql.Row{{sql.OkResult{RowsAffected: 1, Info: plan.UpdateInfo{Matched: 1, Updated: 1, Warnings: 0}}}},
 				},
 			},
 		},

go/libraries/doltcore/sqle/index/index_lookup.go

@@ -66,7 +66,7 @@ func RowIterForProllyRange(ctx *sql.Context, idx DoltIndex, ranges prolly.Range,
 	if covers {
 		return newProllyCoveringIndexIter(ctx, idx, ranges, pkSch, secondary)
 	} else {
-		return newProllyIndexIter(ctx, idx, ranges, primary, secondary)
+		return newProllyIndexIter(ctx, idx, ranges, pkSch, primary, secondary)
 	}
 }
 

go/libraries/doltcore/sqle/index/prolly_fields.go

@@ -28,6 +28,54 @@ import (
 	"github.com/dolthub/dolt/go/store/val"
 )
 
+// todo(andy): this should go in GMS
+func DenormalizeRow(sch sql.Schema, row sql.Row) (sql.Row, error) {
+	var err error
+	for i := range row {
+		if row[i] == nil {
+			continue
+		}
+		switch typ := sch[i].Type.(type) {
+		case sql.DecimalType:
+			row[i] = row[i].(decimal.Decimal).String()
+		case sql.EnumType:
+			row[i], err = typ.Unmarshal(int64(row[i].(uint16)))
+		case sql.SetType:
+			row[i], err = typ.Unmarshal(row[i].(uint64))
+		default:
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+	return row, nil
+}
+
+// todo(andy): this should go in GMS
+func NormalizeRow(sch sql.Schema, row sql.Row) (sql.Row, error) {
+	var err error
+	for i := range row {
+		if row[i] == nil {
+			continue
+		}
+		switch typ := sch[i].Type.(type) {
+		case sql.DecimalType:
+			row[i], err = decimal.NewFromString(row[i].(string))
+		case sql.EnumType:
+			var v int64
+			v, err = typ.Marshal(row[i])
+			row[i] = uint16(v)
+		case sql.SetType:
+			row[i], err = typ.Marshal(row[i])
+		default:
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+	return row, nil
+}
+
 // GetField reads the value from the ith field of the Tuple as an interface{}.
 func GetField(td val.TupleDesc, i int, tup val.Tuple) (v interface{}, err error) {
 	var ok bool
@@ -52,12 +100,10 @@ func GetField(td val.TupleDesc, i int, tup val.Tuple) (v interface{}, err error)
 		v, ok = td.GetFloat32(i, tup)
 	case val.Float64Enc:
 		v, ok = td.GetFloat64(i, tup)
+	case val.Bit64Enc:
+		v, ok = td.GetBit(i, tup)
 	case val.DecimalEnc:
-		var d decimal.Decimal
-		d, ok = td.GetDecimal(i, tup)
-		if ok {
-			v = deserializeDecimal(d)
-		}
+		v, ok = td.GetDecimal(i, tup)
 	case val.YearEnc:
 		v, ok = td.GetYear(i, tup)
 	case val.DateEnc:
@@ -70,6 +116,10 @@ func GetField(td val.TupleDesc, i int, tup val.Tuple) (v interface{}, err error)
 		}
 	case val.DatetimeEnc:
 		v, ok = td.GetDatetime(i, tup)
+	case val.EnumEnc:
+		v, ok = td.GetEnum(i, tup)
+	case val.SetEnc:
+		v, ok = td.GetSet(i, tup)
 	case val.StringEnc:
 		v, ok = td.GetString(i, tup)
 	case val.ByteStringEnc:
@@ -127,12 +177,10 @@ func PutField(tb *val.TupleBuilder, i int, v interface{}) error {
 		tb.PutFloat32(i, v.(float32))
 	case val.Float64Enc:
 		tb.PutFloat64(i, v.(float64))
+	case val.Bit64Enc:
+		tb.PutBit(i, uint64(convUint(v)))
 	case val.DecimalEnc:
-		d, err := serializeDecimal(v.(string))
-		if err != nil {
-			return nil
-		}
-		tb.PutDecimal(i, d)
+		tb.PutDecimal(i, v.(decimal.Decimal))
 	case val.YearEnc:
 		tb.PutYear(i, v.(int16))
 	case val.DateEnc:
@@ -145,6 +193,10 @@ func PutField(tb *val.TupleBuilder, i int, v interface{}) error {
 		tb.PutSqlTime(i, t)
 	case val.DatetimeEnc:
 		tb.PutDatetime(i, v.(time.Time))
+	case val.EnumEnc:
+		tb.PutEnum(i, v.(uint16))
+	case val.SetEnc:
+		tb.PutSet(i, v.(uint64))
 	case val.StringEnc:
 		tb.PutString(i, v.(string))
 	case val.ByteStringEnc:
@@ -220,14 +272,6 @@ func convUint(v interface{}) uint {
 	}
 }
 
-func convJson(v interface{}) (buf []byte, err error) {
-	v, err = sql.JSON.Convert(v)
-	if err != nil {
-		return nil, err
-	}
-	return json.Marshal(v.(sql.JSONDocument).Val)
-}
-
 func deserializeGeometry(buf []byte) (v interface{}) {
 	srid, _, typ := geo.ParseEWKBHeader(buf)
 	buf = buf[geo.EWKBHeaderSize:]
@@ -257,6 +301,18 @@ func serializeGeometry(v interface{}) []byte {
 	}
 }
 
+func convJson(v interface{}) (buf []byte, err error) {
+	v, err = sql.JSON.Convert(v)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(v.(sql.JSONDocument).Val)
+}
+
+func deserializeTime(v int64) (interface{}, error) {
+	return typeinfo.TimeType.ConvertNomsValueToValue(types.Int(v))
+}
+
 func serializeTime(v interface{}) (int64, error) {
 	i, err := typeinfo.TimeType.ConvertValueToNomsValue(nil, nil, v)
 	if err != nil {
@@ -264,15 +320,3 @@ func serializeTime(v interface{}) (int64, error) {
 	}
 	return int64(i.(types.Int)), nil
 }
-
-func deserializeTime(v int64) (interface{}, error) {
-	return typeinfo.TimeType.ConvertNomsValueToValue(types.Int(v))
-}
-
-func serializeDecimal(v interface{}) (decimal.Decimal, error) {
-	return decimal.NewFromString(v.(string))
-}
-
-func deserializeDecimal(v decimal.Decimal) interface{} {
-	return v.String()
-}

go/libraries/doltcore/sqle/index/prolly_fields_test.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/dolthub/go-mysql-server/sql/expression/function"
+	"github.com/shopspring/decimal"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -95,10 +96,15 @@ func TestRoundTripProllyFields(t *testing.T) {
 			typ:   val.Type{Enc: val.Float64Enc},
 			value: float64(-math.Pi),
 		},
+		{
+			name:  "bit",
+			typ:   val.Type{Enc: val.Bit64Enc},
+			value: uint64(42),
+		},
 		{
 			name:  "decimal",
 			typ:   val.Type{Enc: val.DecimalEnc},
-			value: "0.263419374632932747932030573792",
+			value: mustParseDecimal("0.263419374632932747932030573792"),
 		},
 		{
 			name:  "string",
@@ -120,11 +126,11 @@ func TestRoundTripProllyFields(t *testing.T) {
 			typ:   val.Type{Enc: val.DateEnc},
 			value: dateFromTime(time.Now().UTC()),
 		},
-		//{
-		//	name:  "time",
-		//	typ:   val.Type{Enc: val.DateEnc},
-		//	value: dateFromTime(time.Now().UTC()),
-		//},
+		{
+			name:  "time",
+			typ:   val.Type{Enc: val.TimeEnc},
+			value: "11:22:00",
+		},
 		{
 			name:  "datetime",
 			typ:   val.Type{Enc: val.DatetimeEnc},
@@ -207,6 +213,14 @@ func mustParseJson(t *testing.T, s string) sql.JSONDocument {
 	return sql.JSONDocument{Val: v}
 }
 
+func mustParseDecimal(s string) decimal.Decimal {
+	d, err := decimal.NewFromString(s)
+	if err != nil {
+		panic(err)
+	}
+	return d
+}
+
 func dateFromTime(t time.Time) time.Time {
 	y, m, d := t.Year(), t.Month(), t.Day()
 	return time.Date(y, m, d, 0, 0, 0, 0, time.UTC)

go/libraries/doltcore/sqle/index/prolly_index_iter.go

@@ -46,13 +46,20 @@ type prollyIndexIter struct {
 	// keyMap and valMap transform tuples from
 	// primary row storage into sql.Row's
 	keyMap, valMap val.OrdinalMapping
+	sqlSch         sql.Schema
 }
 
 var _ sql.RowIter = prollyIndexIter{}
 var _ sql.RowIter2 = prollyIndexIter{}
 
 // NewProllyIndexIter returns a new prollyIndexIter.
-func newProllyIndexIter(ctx *sql.Context, idx DoltIndex, rng prolly.Range, dprimary, dsecondary durable.Index) (prollyIndexIter, error) {
+func newProllyIndexIter(
+	ctx *sql.Context,
+	idx DoltIndex,
+	rng prolly.Range,
+	pkSch sql.PrimaryKeySchema,
+	dprimary, dsecondary durable.Index,
+) (prollyIndexIter, error) {
 	secondary := durable.ProllyMapFromIndex(dsecondary)
 	indexIter, err := secondary.IterRange(ctx, rng)
 	if err != nil {
@@ -79,6 +86,7 @@ func newProllyIndexIter(ctx *sql.Context, idx DoltIndex, rng prolly.Range, dprim
 		rowChan:   make(chan sql.Row, indexLookupBufSize),
 		keyMap:    km,
 		valMap:    vm,
+		sqlSch:    pkSch.Schema,
 	}
 
 	eg.Go(func() error {
@@ -95,7 +103,7 @@ func (p prollyIndexIter) Next(ctx *sql.Context) (r sql.Row, err error) {
 		select {
 		case r, ok = <-p.rowChan:
 			if ok {
-				return r, nil
+				return DenormalizeRow(p.sqlSch, r)
 			}
 		}
 		if !ok {
@@ -222,6 +230,7 @@ type prollyCoveringIndexIter struct {
 
 	// |keyMap| and |valMap| are both of len ==
 	keyMap, valMap val.OrdinalMapping
+	sqlSch         sql.Schema
 }
 
 var _ sql.RowIter = prollyCoveringIndexIter{}
@@ -251,6 +260,7 @@ func newProllyCoveringIndexIter(ctx *sql.Context, idx DoltIndex, rng prolly.Rang
 		valDesc:   valDesc,
 		keyMap:    keyMap,
 		valMap:    valMap,
+		sqlSch:    pkSch.Schema,
 	}
 
 	return iter, nil
@@ -268,7 +278,7 @@ func (p prollyCoveringIndexIter) Next(ctx *sql.Context) (sql.Row, error) {
 		return nil, err
 	}
 
-	return r, nil
+	return DenormalizeRow(p.sqlSch, r)
 }
 
 func (p prollyCoveringIndexIter) Next2(ctx *sql.Context, f *sql.RowFrame) error {

go/libraries/doltcore/sqle/index/prolly_row_iter.go

@@ -15,7 +15,6 @@
 package index
 
 import (
-	"context"
 	"strings"
 
 	"github.com/dolthub/go-mysql-server/sql"
@@ -52,6 +51,7 @@ var encodingToType [256]query.Type
 type prollyRowIter struct {
 	iter prolly.MapIter
 
+	sqlSch  sql.Schema
 	keyDesc val.TupleDesc
 	valDesc val.TupleDesc
 	keyProj []int
@@ -63,8 +63,8 @@ var _ sql.RowIter = prollyRowIter{}
 var _ sql.RowIter2 = prollyRowIter{}
 
 func NewProllyRowIter(
-	ctx context.Context,
 	sch schema.Schema,
+	schSch sql.Schema,
 	rows prolly.Map,
 	iter prolly.MapIter,
 	projections []string,
@@ -91,6 +91,7 @@ func NewProllyRowIter(
 
 	return prollyRowIter{
 		iter:    iter,
+		sqlSch:  schSch,
 		keyDesc: kd,
 		valDesc: vd,
 		keyProj: keyProj,
@@ -159,8 +160,7 @@ func (it prollyRowIter) Next(ctx *sql.Context) (sql.Row, error) {
 			return nil, err
 		}
 	}
-
-	return row, nil
+	return DenormalizeRow(it.sqlSch, row)
 }
 
 func (it prollyRowIter) Next2(ctx *sql.Context, frame *sql.RowFrame) error {

go/libraries/doltcore/sqle/rows.go

@@ -68,7 +68,7 @@ type doltTableRowIter struct {
 }
 
 // Returns a new row iterator for the table given
-func newRowIterator(ctx context.Context, tbl *doltdb.Table, projCols []string, partition doltTablePartition) (sql.RowIter, error) {
+func newRowIterator(ctx context.Context, tbl *doltdb.Table, sqlSch sql.Schema, projCols []string, partition doltTablePartition) (sql.RowIter, error) {
 	sch, err := tbl.GetSchema(ctx)
 
 	if err != nil {
@@ -76,7 +76,7 @@ func newRowIterator(ctx context.Context, tbl *doltdb.Table, projCols []string, p
 	}
 
 	if types.IsFormat_DOLT_1(tbl.Format()) {
-		return ProllyRowIterFromPartition(ctx, tbl, projCols, partition)
+		return ProllyRowIterFromPartition(ctx, tbl, sqlSch, projCols, partition)
 	}
 
 	if schema.IsKeyless(sch) {
@@ -168,7 +168,13 @@ func (itr *doltTableRowIter) Close(*sql.Context) error {
 	return nil
 }
 
-func ProllyRowIterFromPartition(ctx context.Context, tbl *doltdb.Table, projections []string, partition doltTablePartition) (sql.RowIter, error) {
+func ProllyRowIterFromPartition(
+	ctx context.Context,
+	tbl *doltdb.Table,
+	sqlSch sql.Schema,
+	projections []string,
+	partition doltTablePartition,
+) (sql.RowIter, error) {
 	rows := durable.ProllyMapFromIndex(partition.rowData)
 	sch, err := tbl.GetSchema(ctx)
 	if err != nil {
@@ -183,7 +189,7 @@ func ProllyRowIterFromPartition(ctx context.Context, tbl *doltdb.Table, projecti
 		return nil, err
 	}
 
-	return index.NewProllyRowIter(ctx, sch, rows, iter, projections)
+	return index.NewProllyRowIter(sch, sqlSch, rows, iter, projections)
 }
 
 // TableToRowIter returns a |sql.RowIter| for a full table scan for the given |table|. If
@@ -208,6 +214,7 @@ func TableToRowIter(ctx *sql.Context, table *WritableDoltTable, columns []string
 		end:     NoUpperBound,
 		rowData: data,
 	}
+	sqlSch := table.sqlSch.Schema
 
-	return newRowIterator(ctx, t, columns, p)
+	return newRowIterator(ctx, t, sqlSch, columns, p)
 }

go/libraries/doltcore/sqle/tables.go

@@ -325,7 +325,7 @@ func (t *DoltTable) PartitionRows(ctx *sql.Context, partition sql.Partition) (sq
 		return nil, err
 	}
 
-	return partitionRows(ctx, table, t.projectedCols, partition)
+	return partitionRows(ctx, table, t.sqlSch.Schema, t.projectedCols, partition)
 }
 
 func (t DoltTable) PartitionRows2(ctx *sql.Context, part sql.Partition) (sql.RowIter2, error) {
@@ -334,7 +334,7 @@ func (t DoltTable) PartitionRows2(ctx *sql.Context, part sql.Partition) (sql.Row
 		return nil, err
 	}
 
-	iter, err := partitionRows(ctx, table, t.projectedCols, part)
+	iter, err := partitionRows(ctx, table, t.sqlSch.Schema, t.projectedCols, part)
 	if err != nil {
 		return nil, err
 	}
@@ -342,12 +342,12 @@ func (t DoltTable) PartitionRows2(ctx *sql.Context, part sql.Partition) (sql.Row
 	return iter.(sql.RowIter2), err
 }
 
-func partitionRows(ctx *sql.Context, t *doltdb.Table, projCols []string, partition sql.Partition) (sql.RowIter, error) {
+func partitionRows(ctx *sql.Context, t *doltdb.Table, sqlSch sql.Schema, projCols []string, partition sql.Partition) (sql.RowIter, error) {
 	switch typedPartition := partition.(type) {
 	case doltTablePartition:
-		return newRowIterator(ctx, t, projCols, typedPartition)
+		return newRowIterator(ctx, t, sqlSch, projCols, typedPartition)
 	case index.SinglePartition:
-		return newRowIterator(ctx, t, projCols, doltTablePartition{rowData: typedPartition.RowData, end: NoUpperBound})
+		return newRowIterator(ctx, t, sqlSch, projCols, doltTablePartition{rowData: typedPartition.RowData, end: NoUpperBound})
 	}
 
 	return nil, errors.New("unsupported partition type")
@@ -1274,7 +1274,7 @@ func (t *AlterableDoltTable) ModifyColumn(ctx *sql.Context, columnName string, c
 
 		// Note that we aren't calling the public PartitionRows, because it always gets the table data from the session
 		// root, which hasn't been updated yet
-		rowIter, err := partitionRows(ctx, updatedTable, t.projectedCols, index.SinglePartition{RowData: rowData})
+		rowIter, err := partitionRows(ctx, updatedTable, t.sqlSch.Schema, t.projectedCols, index.SinglePartition{RowData: rowData})
 		if err != nil {
 			return err
 		}

go/libraries/doltcore/sqle/temp_table.go

@@ -152,7 +152,7 @@ func (t *TempTable) PartitionRows(ctx *sql.Context, partition sql.Partition) (sq
 	if t.lookup != nil {
 		return index.RowIterForIndexLookup(ctx, t.table, t.lookup, t.pkSch, nil)
 	} else {
-		return partitionRows(ctx, t.table, nil, partition)
+		return partitionRows(ctx, t.table, t.sqlSchema().Schema, nil, partition)
 	}
 }
 

go/libraries/doltcore/sqle/writer/prolly_fk_indexer.go

@@ -86,6 +86,7 @@ func (n prollyFkIndexer) PartitionRows(ctx *sql.Context, _ sql.Partition) (sql.R
 			rangeIter:  rangeIter,
 			idxToPkMap: idxToPkMap,
 			primary:    primary,
+			sqlSch:     n.writer.sqlSch,
 		}, nil
 	} else {
 		rangeIter, err := idxWriter.(prollyKeylessSecondaryWriter).mut.IterRange(ctx, n.pRange)
@@ -95,6 +96,7 @@ func (n prollyFkIndexer) PartitionRows(ctx *sql.Context, _ sql.Partition) (sql.R
 		return &prollyFkKeylessRowIter{
 			rangeIter: rangeIter,
 			primary:   n.writer.primary.(prollyKeylessWriter),
+			sqlSch:    n.writer.sqlSch,
 		}, nil
 	}
 }
@@ -104,6 +106,7 @@ type prollyFkPkRowIter struct {
 	rangeIter  prolly.MapIter
 	idxToPkMap map[int]int
 	primary    prollyIndexWriter
+	sqlSch     sql.Schema
 }
 
 var _ sql.RowIter = prollyFkPkRowIter{}
@@ -140,7 +143,10 @@ func (iter prollyFkPkRowIter) Next(ctx *sql.Context) (sql.Row, error) {
 		}
 		return nil
 	})
-	return nextRow, err
+	if err != nil {
+		return nil, err
+	}
+	return index.DenormalizeRow(iter.sqlSch, nextRow)
 }
 
 // Close implements the interface sql.RowIter.
@@ -152,6 +158,7 @@ func (iter prollyFkPkRowIter) Close(ctx *sql.Context) error {
 type prollyFkKeylessRowIter struct {
 	rangeIter prolly.MapIter
 	primary   prollyKeylessWriter
+	sqlSch    sql.Schema
 }
 
 var _ sql.RowIter = prollyFkKeylessRowIter{}
@@ -179,7 +186,10 @@ func (iter prollyFkKeylessRowIter) Next(ctx *sql.Context) (sql.Row, error) {
 		}
 		return nil
 	})
-	return nextRow, err
+	if err != nil {
+		return nil, err
+	}
+	return index.DenormalizeRow(iter.sqlSch, nextRow)
 }
 
 // Close implements the interface sql.RowIter.

go/libraries/doltcore/sqle/writer/prolly_table_writer.go

@@ -122,7 +122,11 @@ func getSecondaryKeylessProllyWriters(ctx context.Context, t *doltdb.Table, sqlS
 }
 
 // Insert implements TableWriter.
-func (w *prollyTableWriter) Insert(ctx *sql.Context, sqlRow sql.Row) error {
+func (w *prollyTableWriter) Insert(ctx *sql.Context, sqlRow sql.Row) (err error) {
+	if sqlRow, err = index.NormalizeRow(w.sqlSch, sqlRow); err != nil {
+		return err
+	}
+
 	if err := w.primary.Insert(ctx, sqlRow); err != nil {
 		return err
 	}
@@ -138,7 +142,11 @@ func (w *prollyTableWriter) Insert(ctx *sql.Context, sqlRow sql.Row) error {
 }
 
 // Delete implements TableWriter.
-func (w *prollyTableWriter) Delete(ctx *sql.Context, sqlRow sql.Row) error {
+func (w *prollyTableWriter) Delete(ctx *sql.Context, sqlRow sql.Row) (err error) {
+	if sqlRow, err = index.NormalizeRow(w.sqlSch, sqlRow); err != nil {
+		return err
+	}
+
 	for _, wr := range w.secondary {
 		if err := wr.Delete(ctx, sqlRow); err != nil {
 			return err
@@ -152,6 +160,13 @@ func (w *prollyTableWriter) Delete(ctx *sql.Context, sqlRow sql.Row) error {
 
 // Update implements TableWriter.
 func (w *prollyTableWriter) Update(ctx *sql.Context, oldRow sql.Row, newRow sql.Row) (err error) {
+	if oldRow, err = index.NormalizeRow(w.sqlSch, oldRow); err != nil {
+		return err
+	}
+	if newRow, err = index.NormalizeRow(w.sqlSch, newRow); err != nil {
+		return err
+	}
+
 	for _, wr := range w.secondary {
 		if err := wr.Update(ctx, oldRow, newRow); err != nil {
 			if sql.ErrUniqueKeyViolation.Is(err) {

go/store/prolly/shim.go

@@ -132,20 +132,11 @@ func encodingFromSqlType(typ query.Type) val.Encoding {
 
 	// todo(andy): replace temp encodings
 	switch typ {
-	case query.Type_DECIMAL:
-		return val.DecimalEnc
-	case query.Type_GEOMETRY:
-		return val.GeometryEnc
-	case query.Type_BIT:
-		return val.Uint64Enc
 	case query.Type_BLOB:
-		return val.ByteStringEnc
+		// todo: temporary hack for enginetests
+		return val.StringEnc
 	case query.Type_TEXT:
 		return val.StringEnc
-	case query.Type_ENUM:
-		return val.StringEnc
-	case query.Type_SET:
-		return val.StringEnc
 	case query.Type_JSON:
 		return val.JSONEnc
 	}
@@ -175,6 +166,10 @@ func encodingFromSqlType(typ query.Type) val.Encoding {
 		return val.Float32Enc
 	case query.Type_FLOAT64:
 		return val.Float64Enc
+	case query.Type_BIT:
+		return val.Uint64Enc
+	case query.Type_DECIMAL:
+		return val.DecimalEnc
 	case query.Type_YEAR:
 		return val.YearEnc
 	case query.Type_DATE:
@@ -185,6 +180,10 @@ func encodingFromSqlType(typ query.Type) val.Encoding {
 		return val.DatetimeEnc
 	case query.Type_DATETIME:
 		return val.DatetimeEnc
+	case query.Type_ENUM:
+		return val.EnumEnc
+	case query.Type_SET:
+		return val.SetEnc
 	case query.Type_BINARY:
 		return val.ByteStringEnc
 	case query.Type_VARBINARY:
@@ -193,6 +192,8 @@ func encodingFromSqlType(typ query.Type) val.Encoding {
 		return val.StringEnc
 	case query.Type_VARCHAR:
 		return val.StringEnc
+	case query.Type_GEOMETRY:
+		return val.GeometryEnc
 	default:
 		panic(fmt.Sprintf("unknown encoding %v", typ))
 	}

go/store/val/codec.go

@@ -38,47 +38,49 @@ const (
 type ByteSize uint16
 
 const (
-	int8Size    ByteSize = 1
-	uint8Size   ByteSize = 1
-	int16Size   ByteSize = 2
-	uint16Size  ByteSize = 2
-	int32Size   ByteSize = 4
-	uint32Size  ByteSize = 4
-	int64Size   ByteSize = 8
-	uint64Size  ByteSize = 8
-	float32Size ByteSize = 4
-	float64Size ByteSize = 8
-
-	hash128Size ByteSize = 16
-
+	int8Size     ByteSize = 1
+	uint8Size    ByteSize = 1
+	int16Size    ByteSize = 2
+	uint16Size   ByteSize = 2
+	int32Size    ByteSize = 4
+	uint32Size   ByteSize = 4
+	int64Size    ByteSize = 8
+	uint64Size   ByteSize = 8
+	float32Size  ByteSize = 4
+	float64Size  ByteSize = 8
+	bit64Size    ByteSize = 8
+	hash128Size  ByteSize = 16
 	yearSize     ByteSize = 1
 	dateSize     ByteSize = 4
 	timeSize     ByteSize = 8
 	datetimeSize ByteSize = 8
+	enumSize     ByteSize = 2
+	setSize      ByteSize = 8
 )
 
 type Encoding uint8
 
 // Constant Size Encodings
 const (
-	NullEnc    Encoding = 0
-	Int8Enc    Encoding = 1
-	Uint8Enc   Encoding = 2
-	Int16Enc   Encoding = 3
-	Uint16Enc  Encoding = 4
-	Int32Enc   Encoding = 7
-	Uint32Enc  Encoding = 8
-	Int64Enc   Encoding = 9
-	Uint64Enc  Encoding = 10
-	Float32Enc Encoding = 11
-	Float64Enc Encoding = 12
-
-	Hash128Enc Encoding = 13
-
-	YearEnc     Encoding = 14
-	DateEnc     Encoding = 15
-	TimeEnc     Encoding = 16
-	DatetimeEnc Encoding = 17
+	NullEnc     Encoding = 0
+	Int8Enc     Encoding = 1
+	Uint8Enc    Encoding = 2
+	Int16Enc    Encoding = 3
+	Uint16Enc   Encoding = 4
+	Int32Enc    Encoding = 7
+	Uint32Enc   Encoding = 8
+	Int64Enc    Encoding = 9
+	Uint64Enc   Encoding = 10
+	Float32Enc  Encoding = 11
+	Float64Enc  Encoding = 12
+	Bit64Enc    Encoding = 13
+	Hash128Enc  Encoding = 14
+	YearEnc     Encoding = 15
+	DateEnc     Encoding = 16
+	TimeEnc     Encoding = 17
+	DatetimeEnc Encoding = 18
+	EnumEnc     Encoding = 19
+	SetEnc      Encoding = 20
 
 	sentinel Encoding = 127
 )
@@ -87,25 +89,18 @@ const (
 const (
 	StringEnc     Encoding = 128
 	ByteStringEnc Encoding = 129
-
-	// todo(andy): experimental encodings
-	DecimalEnc  Encoding = 130
-	JSONEnc     Encoding = 131
-	GeometryEnc Encoding = 133
+	DecimalEnc    Encoding = 130
+	JSONEnc       Encoding = 131
+	GeometryEnc   Encoding = 133
 
 	// TODO
-	//  BitEnc
 	//  CharEnc
 	//  VarCharEnc
 	//  TextEnc
 	//  BinaryEnc
 	//  VarBinaryEnc
 	//  BlobEnc
-	//  JSONEnc
-	//  EnumEnc
-	//  SetEnc
 	//  ExpressionEnc
-	//  GeometryEnc
 )
 
 func sizeFromType(t Type) (ByteSize, bool) {
@@ -130,16 +125,22 @@ func sizeFromType(t Type) (ByteSize, bool) {
 		return float32Size, true
 	case Float64Enc:
 		return float64Size, true
+	case Hash128Enc:
+		return hash128Size, true
 	case YearEnc:
 		return yearSize, true
 	case DateEnc:
 		return dateSize, true
-	//case TimeEnc:
-	//	return timeSize, true
+	case TimeEnc:
+		return timeSize, true
 	case DatetimeEnc:
 		return datetimeSize, true
-	case Hash128Enc:
-		return hash128Size, true
+	case EnumEnc:
+		return enumSize, true
+	case SetEnc:
+		return setSize, true
+	case Bit64Enc:
+		return bit64Size, true
 	default:
 		return 0, false
 	}
@@ -370,6 +371,18 @@ func compareFloat64(l, r float64) int {
 	}
 }
 
+func readBit64(val []byte) uint64 {
+	return readUint64(val)
+}
+
+func writeBit64(buf []byte, val uint64) {
+	writeUint64(buf, val)
+}
+
+func compareBit64(l, r uint64) int {
+	return compareUint64(l, r)
+}
+
 func readDecimal(val []byte) decimal.Decimal {
 	e := readInt32(val[:int32Size])
 	s := readInt8(val[int32Size : int32Size+int8Size])
@@ -478,6 +491,30 @@ func compareDatetime(l, r time.Time) int {
 	}
 }
 
+func readEnum(val []byte) uint16 {
+	return readUint16(val)
+}
+
+func writeEnum(buf []byte, val uint16) {
+	writeUint16(buf, val)
+}
+
+func compareEnum(l, r uint16) int {
+	return compareUint16(l, r)
+}
+
+func readSet(val []byte) uint64 {
+	return readUint64(val)
+}
+
+func writeSet(buf []byte, val uint64) {
+	writeUint64(buf, val)
+}
+
+func compareSet(l, r uint64) int {
+	return compareUint64(l, r)
+}
+
 func readString(val []byte) string {
 	return stringFromBytes(readByteString(val))
 }

go/store/val/codec_test.go

@@ -78,6 +78,22 @@ func TestCompare(t *testing.T) {
 			l:   encFloat(1), r: encFloat(0),
 			cmp: 1,
 		},
+		// bit
+		{
+			typ: Type{Enc: Bit64Enc},
+			l:   encBit(0), r: encBit(0),
+			cmp: 0,
+		},
+		{
+			typ: Type{Enc: Bit64Enc},
+			l:   encBit(0), r: encBit(1),
+			cmp: -1,
+		},
+		{
+			typ: Type{Enc: Bit64Enc},
+			l:   encBit(1), r: encBit(0),
+			cmp: 1,
+		},
 		// decimal
 		{
 			typ: Type{Enc: DecimalEnc},
@@ -161,6 +177,38 @@ func TestCompare(t *testing.T) {
 			r:   encDatetime(time.Date(2000, 11, 01, 01, 01, 01, 00, time.UTC)),
 			cmp: -1,
 		},
+		// enum
+		{
+			typ: Type{Enc: EnumEnc},
+			l:   encEnum(0), r: encEnum(0),
+			cmp: 0,
+		},
+		{
+			typ: Type{Enc: EnumEnc},
+			l:   encEnum(0), r: encEnum(1),
+			cmp: -1,
+		},
+		{
+			typ: Type{Enc: EnumEnc},
+			l:   encEnum(1), r: encEnum(0),
+			cmp: 1,
+		},
+		// set
+		{
+			typ: Type{Enc: SetEnc},
+			l:   encSet(0), r: encSet(0),
+			cmp: 0,
+		},
+		{
+			typ: Type{Enc: SetEnc},
+			l:   encSet(0), r: encSet(1),
+			cmp: -1,
+		},
+		{
+			typ: Type{Enc: SetEnc},
+			l:   encSet(1), r: encSet(0),
+			cmp: 1,
+		},
 		// string
 		{
 			typ: Type{Enc: StringEnc},
@@ -231,6 +279,12 @@ func encFloat(f float64) []byte {
 	return buf
 }
 
+func encBit(u uint64) []byte {
+	buf := make([]byte, bit64Size)
+	writeBit64(buf, u)
+	return buf
+}
+
 func encDecimal(d decimal.Decimal) []byte {
 	buf := make([]byte, sizeOfDecimal(d))
 	writeDecimal(buf, d)
@@ -268,6 +322,18 @@ func encDatetime(dt time.Time) []byte {
 	return buf
 }
 
+func encEnum(u uint16) []byte {
+	buf := make([]byte, enumSize)
+	writeEnum(buf, u)
+	return buf
+}
+
+func encSet(u uint64) []byte {
+	buf := make([]byte, setSize)
+	writeSet(buf, u)
+	return buf
+}
+
 func TestCodecRoundTrip(t *testing.T) {
 	t.Run("round trip bool", func(t *testing.T) {
 		roundTripBools(t)
@@ -365,6 +431,14 @@ func roundTripUints(t *testing.T) {
 		zero(buf)
 	}
 
+	buf = make([]byte, enumSize)
+	for _, value := range uintegers {
+		exp := uint16(value)
+		writeEnum(buf, exp)
+		assert.Equal(t, exp, readEnum(buf))
+		zero(buf)
+	}
+
 	buf = make([]byte, uint32Size)
 	uintegers = append(uintegers, math.MaxUint32)
 	for _, value := range uintegers {
@@ -382,6 +456,22 @@ func roundTripUints(t *testing.T) {
 		assert.Equal(t, exp, readUint64(buf))
 		zero(buf)
 	}
+
+	buf = make([]byte, bit64Size)
+	for _, value := range uintegers {
+		exp := uint64(value)
+		writeBit64(buf, exp)
+		assert.Equal(t, exp, readBit64(buf))
+		zero(buf)
+	}
+
+	buf = make([]byte, setSize)
+	for _, value := range uintegers {
+		exp := uint64(value)
+		writeSet(buf, exp)
+		assert.Equal(t, exp, readSet(buf))
+		zero(buf)
+	}
 }
 
 func roundTripFloats(t *testing.T) {

go/store/val/tuple_builder.go

@@ -159,6 +159,21 @@ func (tb *TupleBuilder) PutFloat64(i int, v float64) {
 	tb.pos += float64Size
 }
 
+func (tb *TupleBuilder) PutBit(i int, v uint64) {
+	tb.Desc.expectEncoding(i, Bit64Enc)
+	tb.fields[i] = tb.buf[tb.pos : tb.pos+bit64Size]
+	writeBit64(tb.fields[i], v)
+	tb.pos += bit64Size
+}
+
+func (tb *TupleBuilder) PutDecimal(i int, v decimal.Decimal) {
+	tb.Desc.expectEncoding(i, DecimalEnc)
+	sz := sizeOfDecimal(v)
+	tb.fields[i] = tb.buf[tb.pos : tb.pos+sz]
+	writeDecimal(tb.fields[i], v)
+	tb.pos += sz
+}
+
 // PutYear writes an int16-encoded year to the ith field of the Tuple being built.
 func (tb *TupleBuilder) PutYear(i int, v int16) {
 	tb.Desc.expectEncoding(i, YearEnc)
@@ -189,12 +204,18 @@ func (tb *TupleBuilder) PutDatetime(i int, v time.Time) {
 	tb.pos += datetimeSize
 }
 
-func (tb *TupleBuilder) PutDecimal(i int, v decimal.Decimal) {
-	tb.Desc.expectEncoding(i, DecimalEnc)
-	sz := sizeOfDecimal(v)
-	tb.fields[i] = tb.buf[tb.pos : tb.pos+sz]
-	writeDecimal(tb.fields[i], v)
-	tb.pos += sz
+func (tb *TupleBuilder) PutEnum(i int, v uint16) {
+	tb.Desc.expectEncoding(i, EnumEnc)
+	tb.fields[i] = tb.buf[tb.pos : tb.pos+enumSize]
+	writeEnum(tb.fields[i], v)
+	tb.pos += enumSize
+}
+
+func (tb *TupleBuilder) PutSet(i int, v uint64) {
+	tb.Desc.expectEncoding(i, SetEnc)
+	tb.fields[i] = tb.buf[tb.pos : tb.pos+setSize]
+	writeSet(tb.fields[i], v)
+	tb.pos += setSize
 }
 
 // PutString writes a string to the ith field of the Tuple being built.

go/store/val/tuple_compare.go

@@ -90,6 +90,10 @@ func compare(typ Type, left, right []byte) int {
 		return compareFloat32(readFloat32(left), readFloat32(right))
 	case Float64Enc:
 		return compareFloat64(readFloat64(left), readFloat64(right))
+	case Bit64Enc:
+		return compareBit64(readBit64(left), readBit64(right))
+	case DecimalEnc:
+		return compareDecimal(readDecimal(left), readDecimal(right))
 	case YearEnc:
 		return compareYear(readYear(left), readYear(right))
 	case DateEnc:
@@ -98,8 +102,10 @@ func compare(typ Type, left, right []byte) int {
 		return compareTime(readTime(left), readTime(right))
 	case DatetimeEnc:
 		return compareDatetime(readDatetime(left), readDatetime(right))
-	case DecimalEnc:
-		return compareDecimal(readDecimal(left), readDecimal(right))
+	case EnumEnc:
+		return compareEnum(readEnum(left), readEnum(right))
+	case SetEnc:
+		return compareSet(readSet(left), readSet(right))
 	case StringEnc:
 		return compareString(readString(left), readString(right))
 	case ByteStringEnc:

go/store/val/tuple_descriptor.go

@@ -241,6 +241,17 @@ func (td TupleDesc) GetFloat64(i int, tup Tuple) (v float64, ok bool) {
 	return
 }
 
+// GetBit reads a uint64 from the ith field of the Tuple.
+// If the ith field is NULL, |ok| is set to false.
+func (td TupleDesc) GetBit(i int, tup Tuple) (v uint64, ok bool) {
+	td.expectEncoding(i, Bit64Enc)
+	b := td.GetField(i, tup)
+	if b != nil {
+		v, ok = readBit64(b), true
+	}
+	return
+}
+
 // GetDecimal reads a float64 from the ith field of the Tuple.
 // If the ith field is NULL, |ok| is set to false.
 func (td TupleDesc) GetDecimal(i int, tup Tuple) (v decimal.Decimal, ok bool) {
@@ -296,6 +307,28 @@ func (td TupleDesc) GetDatetime(i int, tup Tuple) (v time.Time, ok bool) {
 	return
 }
 
+// GetEnum reads a uin16 from the ith field of the Tuple.
+// If the ith field is NULL, |ok| is set to false.
+func (td TupleDesc) GetEnum(i int, tup Tuple) (v uint16, ok bool) {
+	td.expectEncoding(i, EnumEnc)
+	b := td.GetField(i, tup)
+	if b != nil {
+		v, ok = readEnum(b), true
+	}
+	return
+}
+
+// GetSet reads a uint64 from the ith field of the Tuple.
+// If the ith field is NULL, |ok| is set to false.
+func (td TupleDesc) GetSet(i int, tup Tuple) (v uint64, ok bool) {
+	td.expectEncoding(i, SetEnc)
+	b := td.GetField(i, tup)
+	if b != nil {
+		v, ok = readSet(b), true
+	}
+	return
+}
+
 // GetString reads a string from the ith field of the Tuple.
 // If the ith field is NULL, |ok| is set to false.
 func (td TupleDesc) GetString(i int, tup Tuple) (v string, ok bool) {
@@ -423,19 +456,30 @@ func formatValue(enc Encoding, value []byte) string {
 	case Float64Enc:
 		v := readFloat64(value)
 		return fmt.Sprintf("%f", v)
+	case Bit64Enc:
+		v := readUint64(value)
+		return strconv.FormatUint(v, 10)
+	case DecimalEnc:
+		v := readDecimal(value)
+		return v.String()
 	case YearEnc:
 		v := readYear(value)
 		return strconv.Itoa(int(v))
 	case DateEnc:
 		v := readDate(value)
 		return v.Format("2006-01-02")
-	//case TimeEnc:
-	//	// todo(andy)
-	//	v := readTime(value)
-	//	return v
+	case TimeEnc:
+		v := readTime(value)
+		return strconv.FormatInt(v, 10)
 	case DatetimeEnc:
 		v := readDatetime(value)
 		return v.Format(time.RFC3339)
+	case EnumEnc:
+		v := readEnum(value)
+		return strconv.Itoa(int(v))
+	case SetEnc:
+		v := readSet(value)
+		return strconv.FormatUint(v, 10)
 	case StringEnc:
 		return readString(value)
 	case ByteStringEnc: