diff --git a/go/libraries/doltcore/doltdb/durable/index.go b/go/libraries/doltcore/doltdb/durable/index.go index 97fe9e411b..87e0325f6c 100644 --- a/go/libraries/doltcore/doltdb/durable/index.go +++ b/go/libraries/doltcore/doltdb/durable/index.go @@ -49,6 +49,10 @@ type Index interface { // AddColumnToRows adds the column given to the rows data and returns the resulting rows. // The |newCol| is present in |newSchema|. AddColumnToRows(ctx context.Context, newCol string, newSchema schema.Schema) (Index, error) + + // Returns the serialized bytes of the (top of the) index. + // Non-public. Used for flatbuffers Table persistence. + bytes() ([]byte, error) } // IndexSet stores a collection secondary Indexes. @@ -178,6 +182,15 @@ func (i nomsIndex) Format() *types.NomsBinFormat { return i.vrw.Format() } +// bytes implements Index. +func (i nomsIndex) bytes() ([]byte, error) { + rowschunk, err := types.EncodeValue(i.index, i.vrw.Format()) + if err != nil { + return nil, err + } + return rowschunk.Data(), nil +} + func (i nomsIndex) AddColumnToRows(ctx context.Context, newCol string, newSchema schema.Schema) (Index, error) { // no-op for noms indexes because of tag-based mapping return i, nil @@ -219,6 +232,11 @@ func (i prollyIndex) Format() *types.NomsBinFormat { return i.index.Format() } +// bytes implements Index. +func (i prollyIndex) bytes() ([]byte, error) { + return []byte(prolly.ValueFromMap(i.index).(types.TupleRowStorage)), nil +} + var _ Index = prollyIndex{} func (i prollyIndex) AddColumnToRows(ctx context.Context, newCol string, newSchema schema.Schema) (Index, error) { @@ -286,7 +304,7 @@ func (i prollyIndex) AddColumnToRows(ctx context.Context, newCol string, newSche // NewIndexSet returns an empty IndexSet. func NewIndexSet(ctx context.Context, vrw types.ValueReadWriter) IndexSet { - if vrw.Format() == types.Format_DOLT_DEV { + if vrw.Format().UsesFlatbuffers() { builder := flatbuffers.NewBuilder(24) serial.RefMapStart(builder) builder.Finish(serial.RefMapEnd(builder)) diff --git a/go/libraries/doltcore/doltdb/durable/table.go b/go/libraries/doltcore/doltdb/durable/table.go index 51e80f2a74..61f87027af 100644 --- a/go/libraries/doltcore/doltdb/durable/table.go +++ b/go/libraries/doltcore/doltdb/durable/table.go @@ -30,6 +30,7 @@ import ( "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/pool" + "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/types" ) @@ -138,7 +139,7 @@ func NewNomsTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.Sch // NewTable returns a new Table. func NewTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema, rows Index, indexes IndexSet, autoIncVal types.Value) (Table, error) { - if vrw.Format() == types.Format_DOLT_DEV { + if vrw.Format().UsesFlatbuffers() { return newDoltDevTable(ctx, vrw, sch, rows, indexes, autoIncVal) } @@ -191,7 +192,7 @@ func TableFromAddr(ctx context.Context, vrw types.ValueReadWriter, addr hash.Has return nil, err } - if vrw.Format() != types.Format_DOLT_DEV { + if !vrw.Format().UsesFlatbuffers() { st, ok := val.(types.Struct) if !ok { err = errors.New("table ref is unexpected noms value") @@ -655,12 +656,10 @@ func newDoltDevTable(ctx context.Context, vrw types.ValueReadWriter, sch schema. } schemaAddr := schemaRef.TargetHash() - rowsmap := rows.(nomsIndex).index - rowschunk, err := types.EncodeValue(rowsmap, vrw.Format()) + rowsbytes, err := rows.bytes() if err != nil { return nil, err } - rowsbytes := rowschunk.Data() if indexes == nil { indexes = NewIndexSet(ctx, vrw) @@ -727,21 +726,28 @@ func (t doltDevTable) SetSchema(ctx context.Context, sch schema.Schema) (Table, func (t doltDevTable) GetTableRows(ctx context.Context) (Index, error) { rowbytes := t.msg.PrimaryIndexBytes() - rowchunk := chunks.NewChunk(rowbytes) - tv, err := types.DecodeValue(rowchunk, t.vrw) - if err != nil { - return nil, err + if t.vrw.Format() == types.Format_DOLT_DEV { + rowchunk := chunks.NewChunk(rowbytes) + tv, err := types.DecodeValue(rowchunk, t.vrw) + if err != nil { + return nil, err + } + return IndexFromNomsMap(tv.(types.Map), t.vrw), nil + } else { + sch, err := t.GetSchema(ctx) + if err != nil { + return nil, err + } + m := prolly.MapFromValue(types.TupleRowStorage(rowbytes), sch, t.vrw) + return IndexFromProllyMap(m), nil } - return IndexFromNomsMap(tv.(types.Map), t.vrw), nil } func (t doltDevTable) SetTableRows(ctx context.Context, rows Index) (Table, error) { - rowsmap := rows.(nomsIndex).index - rowschunk, err := types.EncodeValue(rowsmap, t.vrw.Format()) + rowsbytes, err := rows.bytes() if err != nil { return nil, err } - rowsbytes := rowschunk.Data() fields := t.fields() fields.rows = rowsbytes diff --git a/go/libraries/doltcore/doltdb/root_val.go b/go/libraries/doltcore/doltdb/root_val.go index af053d5fb2..6cb029f68a 100644 --- a/go/libraries/doltcore/doltdb/root_val.go +++ b/go/libraries/doltcore/doltdb/root_val.go @@ -250,7 +250,7 @@ func (r nomsRvStorage) nomsValue() types.Value { func newRootValue(vrw types.ValueReadWriter, v types.Value) (*RootValue, error) { var storage rvStorage - if vrw.Format() == types.Format_DOLT_DEV { + if vrw.Format().UsesFlatbuffers() { srv := serial.GetRootAsRootValue([]byte(v.(types.SerialMessage)), 0) storage = fbRvStorage{srv} } else { @@ -278,7 +278,7 @@ func newRootValue(vrw types.ValueReadWriter, v types.Value) (*RootValue, error) } func isRootValue(nbf *types.NomsBinFormat, val types.Value) bool { - if nbf == types.Format_DOLT_DEV { + if nbf.UsesFlatbuffers() { if sm, ok := val.(types.SerialMessage); ok { return string(serial.GetFileID([]byte(sm))) == serial.RootValueFileID } @@ -291,7 +291,7 @@ func isRootValue(nbf *types.NomsBinFormat, val types.Value) bool { } func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter) (*RootValue, error) { - if vrw.Format() == types.Format_DOLT_DEV { + if vrw.Format().UsesFlatbuffers() { builder := flatbuffers.NewBuilder(80) var empty hash.Hash serial.RefMapStart(builder) diff --git a/go/store/datas/commit.go b/go/store/datas/commit.go index 955b707aa2..e0d688bb20 100644 --- a/go/store/datas/commit.go +++ b/go/store/datas/commit.go @@ -177,7 +177,7 @@ func newCommitForValue(ctx context.Context, vrw types.ValueReadWriter, v types.V opts.Meta = &CommitMeta{} } - if vrw.Format() == types.Format_DOLT_DEV { + if vrw.Format().UsesFlatbuffers() { r, err := vrw.WriteValue(ctx, v) if err != nil { return nil, err @@ -241,7 +241,7 @@ func newCommitForValue(ctx context.Context, vrw types.ValueReadWriter, v types.V } func commitPtr(nbf *types.NomsBinFormat, v types.Value, r *types.Ref) (*Commit, error) { - if nbf == types.Format_DOLT_DEV { + if nbf.UsesFlatbuffers() { bs := []byte(v.(types.SerialMessage)) height := serial.GetRootAsCommit(bs, 0).Height() var addr hash.Hash diff --git a/go/store/datas/database_common.go b/go/store/datas/database_common.go index ed92ba1a65..8e6b790f79 100644 --- a/go/store/datas/database_common.go +++ b/go/store/datas/database_common.go @@ -283,7 +283,7 @@ func (db *database) Datasets(ctx context.Context) (DatasetsMap, error) { return nil, err } - if db.Format() == types.Format_DOLT_DEV { + if db.Format().UsesFlatbuffers() { rm, err := db.loadDatasetsRefmap(ctx, rootHash) if err != nil { return nil, err @@ -792,7 +792,7 @@ func (db *database) update(ctx context.Context, var newRootHash hash.Hash - if db.Format() == types.Format_DOLT_DEV { + if db.Format().UsesFlatbuffers() { datasets, err := db.loadDatasetsRefmap(ctx, root) if err != nil { return err diff --git a/go/store/datas/database_test.go b/go/store/datas/database_test.go index c00be5e87d..7856a7cb8f 100644 --- a/go/store/datas/database_test.go +++ b/go/store/datas/database_test.go @@ -278,7 +278,7 @@ func mustNomsMap(t *testing.T, dsm DatasetsMap) types.Map { } func (suite *DatabaseSuite) TestDatasetsMapType() { - if suite.db.Format() == types.Format_DOLT_DEV { + if suite.db.Format().UsesFlatbuffers() { suite.T().Skip() } diff --git a/go/store/datas/tag.go b/go/store/datas/tag.go index e45c373fa3..1890963d2a 100644 --- a/go/store/datas/tag.go +++ b/go/store/datas/tag.go @@ -52,7 +52,7 @@ type TagOptions struct { // persists it, and returns its addr. Also returns a types.Ref to the tag, if // the format for |db| is noms. func newTag(ctx context.Context, db *database, commitAddr hash.Hash, meta *TagMeta) (hash.Hash, types.Ref, error) { - if db.Format() != types.Format_DOLT_DEV { + if !db.Format().UsesFlatbuffers() { commitSt, err := db.ReadValue(ctx, commitAddr) if err != nil { return hash.Hash{}, types.Ref{}, err diff --git a/go/store/datas/tag_test.go b/go/store/datas/tag_test.go index 684aa2ef86..2852785226 100644 --- a/go/store/datas/tag_test.go +++ b/go/store/datas/tag_test.go @@ -39,7 +39,7 @@ func TestNewTag(t *testing.T) { db := NewDatabase(storage.NewViewWithDefaultFormat()).(*database) defer db.Close() - if db.Format() == types.Format_DOLT_DEV { + if db.Format().UsesFlatbuffers() { t.Skip() } diff --git a/go/store/datas/workingset.go b/go/store/datas/workingset.go index 634b91200f..1db83c023c 100755 --- a/go/store/datas/workingset.go +++ b/go/store/datas/workingset.go @@ -99,7 +99,7 @@ type WorkingSetSpec struct { // ``` // where M is a struct type and R is a ref type. func newWorkingSet(ctx context.Context, db *database, meta *WorkingSetMeta, workingRef, stagedRef types.Ref, mergeState *MergeState) (hash.Hash, types.Ref, error) { - if db.Format() == types.Format_DOLT_DEV { + if db.Format().UsesFlatbuffers() { stagedAddr := stagedRef.TargetHash() data := workingset_flatbuffer(workingRef.TargetHash(), &stagedAddr, mergeState, meta) @@ -191,7 +191,7 @@ func workingset_flatbuffer(working hash.Hash, staged *hash.Hash, mergeState *Mer } func NewMergeState(ctx context.Context, vrw types.ValueReadWriter, preMergeWorking types.Ref, commit *Commit) (*MergeState, error) { - if vrw.Format() == types.Format_DOLT_DEV { + if vrw.Format().UsesFlatbuffers() { ms := &MergeState{ preMergeWorkingAddr: new(hash.Hash), fromCommitAddr: new(hash.Hash), diff --git a/go/store/types/format.go b/go/store/types/format.go index d8a0dc8a0d..f93727a919 100644 --- a/go/store/types/format.go +++ b/go/store/types/format.go @@ -120,3 +120,7 @@ func (nbf *NomsBinFormat) VersionString() string { panic("unrecognized NomsBinFormat tag value") } } + +func (nbf *NomsBinFormat) UsesFlatbuffers() bool { + return nbf.tag == formatTag_DOLT_1 || nbf.tag == formatTag_DOLT_DEV +} diff --git a/go/store/types/serial_message.go b/go/store/types/serial_message.go index 038665c89e..4b98d92bdc 100644 --- a/go/store/types/serial_message.go +++ b/go/store/types/serial_message.go @@ -268,13 +268,16 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { mapbytes := msg.PrimaryIndexBytes() - dec := newValueDecoder(mapbytes, nil) - v, err := dec.readValue(nbf) - if err != nil { - return err + if nbf == Format_DOLT_DEV { + dec := newValueDecoder(mapbytes, nil) + v, err := dec.readValue(nbf) + if err != nil { + return err + } + return v.walkRefs(nbf, cb) + } else { + return TupleRowStorage(mapbytes).walkRefs(nbf, cb) } - - return v.walkRefs(nbf, cb) case serial.CommitFileID: parents, err := SerialCommitParentAddrs(nbf, sm) if err != nil {