From 4f3f4e17af03b11408c8bcb99aaa70a7633af82a Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Fri, 22 Jul 2022 11:47:36 -0700 Subject: [PATCH] go/store/types: SerialMessage: Unify TupleRowStorage and SerialMessage. Get rid of TupleRowStorage and use SerialMessage everywhere. In turn, change SerialMessage so that it carries its kind and size prefix as part of itself. --- go/cmd/dolt/commands/roots.go | 5 +- go/libraries/doltcore/doltdb/durable/index.go | 2 +- go/libraries/doltcore/doltdb/durable/table.go | 16 ++- .../doltdb/foreign_key_serialization.go | 12 +- go/libraries/doltcore/doltdb/root_val.go | 20 +-- .../doltcore/schema/encoding/serialization.go | 9 +- .../doltcore/schema/typeinfo/typeconverter.go | 2 +- go/libraries/doltcore/sqle/sqlselect_test.go | 2 +- go/store/cmd/noms/noms_root_test.go | 4 +- go/store/cmd/noms/noms_show.go | 19 ++- go/store/datas/commit.go | 26 ++-- go/store/datas/commit_closure.go | 7 +- go/store/datas/commit_test.go | 19 +-- go/store/datas/dataset.go | 12 +- go/store/datas/refmap.go | 16 +-- go/store/datas/tag.go | 8 +- go/store/datas/workingset.go | 9 +- go/store/prolly/commit_closure_test.go | 2 +- go/store/prolly/message/address_map.go | 26 ++-- go/store/prolly/message/commit_closure.go | 18 +-- go/store/prolly/message/message.go | 14 +- go/store/prolly/message/prolly_map.go | 16 +-- go/store/prolly/shim/shim.go | 2 +- go/store/prolly/tree/content_address_test.go | 9 +- go/store/prolly/tree/node.go | 2 +- go/store/prolly/tree/node_test.go | 2 +- go/store/types/codec.go | 2 +- go/store/types/noms_kind.go | 64 +++++---- go/store/types/path.go | 6 +- go/store/types/serial_message.go | 91 +++++++------ go/store/types/tuplerowstorage.go | 122 ------------------ 31 files changed, 233 insertions(+), 331 deletions(-) delete mode 100755 go/store/types/tuplerowstorage.go diff --git a/go/cmd/dolt/commands/roots.go b/go/cmd/dolt/commands/roots.go index 360bb184bc..0a305f849e 100644 --- a/go/cmd/dolt/commands/roots.go +++ b/go/cmd/dolt/commands/roots.go @@ -34,6 +34,7 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/chunks" "github.com/dolthub/dolt/go/store/nbs" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/spec" "github.com/dolthub/dolt/go/store/types" @@ -151,8 +152,8 @@ func (cmd RootsCmd) processTableFile(ctx context.Context, path string, modified cli.Println() } } else if sm, ok := value.(types.SerialMessage); ok { - if serial.GetFileID([]byte(sm)) == serial.StoreRootFileID { - msg := serial.GetRootAsStoreRoot([]byte(sm), 0) + if serial.GetFileID(sm[message.MessagePrefixSz:]) == serial.StoreRootFileID { + msg := serial.GetRootAsStoreRoot([]byte(sm), message.MessagePrefixSz) ambytes := msg.AddressMapBytes() node := tree.NodeFromBytes(ambytes) err := tree.OutputAddressMapNode(cli.OutStream, node) diff --git a/go/libraries/doltcore/doltdb/durable/index.go b/go/libraries/doltcore/doltdb/durable/index.go index c7c5b4390a..58b954cf24 100644 --- a/go/libraries/doltcore/doltdb/durable/index.go +++ b/go/libraries/doltcore/doltdb/durable/index.go @@ -250,7 +250,7 @@ func (i prollyIndex) Format() *types.NomsBinFormat { // bytes implements Index. func (i prollyIndex) bytes() ([]byte, error) { - return []byte(shim.ValueFromMap(i.index).(types.TupleRowStorage)), nil + return []byte(shim.ValueFromMap(i.index).(types.SerialMessage)), nil } var _ Index = prollyIndex{} diff --git a/go/libraries/doltcore/doltdb/durable/table.go b/go/libraries/doltcore/doltdb/durable/table.go index a0753a2a4a..430cdd6aea 100644 --- a/go/libraries/doltcore/doltdb/durable/table.go +++ b/go/libraries/doltcore/doltdb/durable/table.go @@ -30,6 +30,7 @@ import ( "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/pool" "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/prolly/shim" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" @@ -194,11 +195,12 @@ func TableFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS err = errors.New("table ref is unexpected noms value; not SerialMessage") return nil, err } - if serial.GetFileID([]byte(sm)) != serial.TableFileID { - err = errors.New("table ref is unexpected noms value; GetFileID == " + serial.GetFileID([]byte(sm))) + id := serial.GetFileID(sm[message.MessagePrefixSz:]) + if id != serial.TableFileID { + err = errors.New("table ref is unexpected noms value; GetFileID == " + id) return nil, err } - return doltDevTable{vrw, ns, serial.GetRootAsTable([]byte(sm), 0)}, nil + return doltDevTable{vrw, ns, serial.GetRootAsTable([]byte(sm), message.MessagePrefixSz)}, nil } } @@ -745,7 +747,7 @@ func (fields serialTableFields) write() *serial.Table { builder := flatbuffers.NewBuilder(1024) indexesam := fields.indexes - indexesbytes := []byte(tree.ValueFromNode(indexesam.Node()).(types.TupleRowStorage)) + indexesbytes := []byte(tree.ValueFromNode(indexesam.Node()).(types.SerialMessage)) schemaoff := builder.CreateByteVector(fields.schema) rowsoff := builder.CreateByteVector(fields.rows) @@ -772,8 +774,8 @@ func (fields serialTableFields) write() *serial.Table { serial.TableAddConflicts(builder, conflictsoff) serial.TableAddViolations(builder, violationsoff) serial.TableAddArtifacts(builder, artifactsoff) - builder.FinishWithFileIdentifier(serial.TableEnd(builder), []byte(serial.TableFileID)) - return serial.GetRootAsTable(builder.FinishedBytes(), 0) + bs := message.FinishMessage(builder, serial.TableEnd(builder), []byte(serial.TableFileID)) + return serial.GetRootAsTable(bs, message.MessagePrefixSz) } func newDoltDevTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows Index, indexes IndexSet, autoIncVal types.Value) (Table, error) { @@ -871,7 +873,7 @@ func (t doltDevTable) GetTableRows(ctx context.Context) (Index, error) { if err != nil { return nil, err } - m := shim.MapFromValue(types.TupleRowStorage(rowbytes), sch, t.ns) + m := shim.MapFromValue(types.SerialMessage(rowbytes), sch, t.ns) return IndexFromProllyMap(m), nil } } diff --git a/go/libraries/doltcore/doltdb/foreign_key_serialization.go b/go/libraries/doltcore/doltdb/foreign_key_serialization.go index 8a9b910398..131563dc51 100644 --- a/go/libraries/doltcore/doltdb/foreign_key_serialization.go +++ b/go/libraries/doltcore/doltdb/foreign_key_serialization.go @@ -22,6 +22,7 @@ import ( "github.com/dolthub/dolt/go/gen/fb/serial" "github.com/dolthub/dolt/go/store/marshal" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/types" ) @@ -84,11 +85,11 @@ func serializeNomsForeignKeys(ctx context.Context, vrw types.ValueReadWriter, fk // deserializeNomsForeignKeys returns a new ForeignKeyCollection using the provided map returned previously by GetMap. func deserializeFlatbufferForeignKeys(msg types.SerialMessage) (*ForeignKeyCollection, error) { - if serial.GetFileID(msg) != serial.ForeignKeyCollectionFileID { + if serial.GetFileID(msg[message.MessagePrefixSz:]) != serial.ForeignKeyCollectionFileID { return nil, fmt.Errorf("expect Serial Message with ForeignKeyCollectionFileID") } - c := serial.GetRootAsForeignKeyCollection(msg, 0) + c := serial.GetRootAsForeignKeyCollection(msg, message.MessagePrefixSz) collection := &ForeignKeyCollection{ foreignKeys: make(map[string]ForeignKey, c.ForeignKeysLength()), } @@ -203,8 +204,7 @@ func serializeFlatbufferForeignKeys(fkc *ForeignKeyCollection) types.SerialMessa serial.ForeignKeyCollectionStart(b) serial.ForeignKeyCollectionAddForeignKeys(b, vec) o := serial.ForeignKeyCollectionEnd(b) - b.FinishWithFileIdentifier(o, []byte(serial.ForeignKeyCollectionFileID)) - return types.SerialMessage(b.FinishedBytes()) + return []byte(message.FinishMessage(b, o, []byte(serial.ForeignKeyCollectionFileID))) } func serializeStringVector(b *fb.Builder, s []string) fb.UOffsetT { @@ -228,9 +228,9 @@ func serializeUint64Vector(b *fb.Builder, u []uint64) fb.UOffsetT { } func emptyForeignKeyCollection(msg types.SerialMessage) bool { - if serial.GetFileID(msg) != serial.ForeignKeyCollectionFileID { + if serial.GetFileID(msg[message.MessagePrefixSz:]) != serial.ForeignKeyCollectionFileID { return false } - c := serial.GetRootAsForeignKeyCollection(msg, 0) + c := serial.GetRootAsForeignKeyCollection(msg, message.MessagePrefixSz) return c.ForeignKeysLength() == 0 } diff --git a/go/libraries/doltcore/doltdb/root_val.go b/go/libraries/doltcore/doltdb/root_val.go index df0e2612bb..a859742698 100644 --- a/go/libraries/doltcore/doltdb/root_val.go +++ b/go/libraries/doltcore/doltdb/root_val.go @@ -29,6 +29,7 @@ import ( "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/prolly/shim" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" @@ -255,7 +256,7 @@ func newRootValue(vrw types.ValueReadWriter, ns tree.NodeStore, v types.Value) ( var storage rvStorage if vrw.Format().UsesFlatbuffers() { - srv := serial.GetRootAsRootValue([]byte(v.(types.SerialMessage)), 0) + srv := serial.GetRootAsRootValue([]byte(v.(types.SerialMessage)), message.MessagePrefixSz) storage = fbRvStorage{srv} } else { st, ok := v.(types.Struct) @@ -306,7 +307,7 @@ func decodeRootNomsValue(vrw types.ValueReadWriter, ns tree.NodeStore, val types func isRootValue(nbf *types.NomsBinFormat, val types.Value) bool { if nbf.UsesFlatbuffers() { if sm, ok := val.(types.SerialMessage); ok { - return string(serial.GetFileID([]byte(sm))) == serial.RootValueFileID + return string(serial.GetFileID(sm[message.MessagePrefixSz:])) == serial.RootValueFileID } } else { if st, ok := val.(types.Struct); ok { @@ -321,7 +322,7 @@ func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter, ns tree.Node builder := flatbuffers.NewBuilder(80) emptyam := prolly.NewEmptyAddressMap(ns) - ambytes := []byte(tree.ValueFromNode(emptyam.Node()).(types.TupleRowStorage)) + ambytes := []byte(tree.ValueFromNode(emptyam.Node()).(types.SerialMessage)) tablesoff := builder.CreateByteVector(ambytes) var empty hash.Hash @@ -332,8 +333,7 @@ func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter, ns tree.Node serial.RootValueAddTables(builder, tablesoff) serial.RootValueAddForeignKeyAddr(builder, fkoff) serial.RootValueAddSuperSchemasAddr(builder, ssoff) - builder.FinishWithFileIdentifier(serial.RootValueEnd(builder), []byte(serial.RootValueFileID)) - bs := builder.FinishedBytes() + bs := message.FinishMessage(builder, serial.RootValueEnd(builder), []byte(serial.RootValueFileID)) return newRootValue(vrw, ns, types.SerialMessage(bs)) } @@ -1143,7 +1143,7 @@ func (r fbRvStorage) GetFeatureVersion() (FeatureVersion, bool, error) { func (r fbRvStorage) getAddressMap(vrw types.ValueReadWriter, ns tree.NodeStore) prolly.AddressMap { tbytes := r.srv.TablesBytes() - node := shim.NodeFromValue(types.TupleRowStorage(tbytes)) + node := shim.NodeFromValue(types.SerialMessage(tbytes)) return prolly.NewAddressMap(node, ns) } @@ -1258,7 +1258,7 @@ func (r fbRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWrite return nil, err } - ambytes := []byte(tree.ValueFromNode(am.Node()).(types.TupleRowStorage)) + ambytes := []byte(tree.ValueFromNode(am.Node()).(types.SerialMessage)) tablesoff := builder.CreateByteVector(ambytes) fkoff := builder.CreateByteVector(r.srv.ForeignKeyAddrBytes()) @@ -1268,9 +1268,9 @@ func (r fbRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWrite serial.RootValueAddTables(builder, tablesoff) serial.RootValueAddForeignKeyAddr(builder, fkoff) serial.RootValueAddSuperSchemasAddr(builder, ssoff) - builder.FinishWithFileIdentifier(serial.RootValueEnd(builder), []byte(serial.RootValueFileID)) - bs := builder.FinishedBytes() - return fbRvStorage{serial.GetRootAsRootValue(bs, 0)}, nil + + bs := message.FinishMessage(builder, serial.RootValueEnd(builder), []byte(serial.RootValueFileID)) + return fbRvStorage{serial.GetRootAsRootValue(bs, message.MessagePrefixSz)}, nil } func (r fbRvStorage) SetForeignKeyMap(ctx context.Context, vrw types.ValueReadWriter, v types.Value) (rvStorage, error) { diff --git a/go/libraries/doltcore/schema/encoding/serialization.go b/go/libraries/doltcore/schema/encoding/serialization.go index 47d4799fc2..078d8d2c8b 100644 --- a/go/libraries/doltcore/schema/encoding/serialization.go +++ b/go/libraries/doltcore/schema/encoding/serialization.go @@ -25,6 +25,7 @@ import ( "github.com/dolthub/dolt/go/gen/fb/serial" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/types" ) @@ -62,8 +63,8 @@ func serializeSchemaAsFlatbuffer(sch schema.Schema) ([]byte, error) { serial.TableSchemaAddSecondaryIndexes(b, indexes) serial.TableSchemaAddChecks(b, checks) root := serial.TableSchemaEnd(b) - b.FinishWithFileIdentifier(root, []byte(serial.TableSchemaFileID)) - return b.FinishedBytes(), nil + bs := message.FinishMessage(b, root, []byte(serial.TableSchemaFileID)) + return bs, nil } // DeserializeSchema deserializes a schema.Schema from a serial.Message. @@ -75,8 +76,8 @@ func DeserializeSchema(ctx context.Context, nbf *types.NomsBinFormat, v types.Va } func deserializeSchemaFromFlatbuffer(ctx context.Context, buf []byte) (schema.Schema, error) { - assertTrue(serial.GetFileID(buf) == serial.TableSchemaFileID) - s := serial.GetRootAsTableSchema(buf, 0) + assertTrue(serial.GetFileID(buf[message.MessagePrefixSz:]) == serial.TableSchemaFileID) + s := serial.GetRootAsTableSchema(buf, message.MessagePrefixSz) cols, err := deserializeColumns(ctx, s) if err != nil { diff --git a/go/libraries/doltcore/schema/typeinfo/typeconverter.go b/go/libraries/doltcore/schema/typeinfo/typeconverter.go index 41c3731bd8..1c876dac3f 100644 --- a/go/libraries/doltcore/schema/typeinfo/typeconverter.go +++ b/go/libraries/doltcore/schema/typeinfo/typeconverter.go @@ -131,7 +131,7 @@ func wrapConvertValueToNomsValue( vInt = float64(val) case types.InlineBlob: vInt = *(*string)(unsafe.Pointer(&val)) - case types.TupleRowStorage: + case types.SerialMessage: vInt = *(*string)(unsafe.Pointer(&val)) case types.Int: vInt = int64(val) diff --git a/go/libraries/doltcore/sqle/sqlselect_test.go b/go/libraries/doltcore/sqle/sqlselect_test.go index 154938692b..4fcbe3f4b6 100644 --- a/go/libraries/doltcore/sqle/sqlselect_test.go +++ b/go/libraries/doltcore/sqle/sqlselect_test.go @@ -87,7 +87,7 @@ func LoadedLocalLocation() *time.Location { func BasicSelectTests() []SelectTest { headCommitHash := "73hc2robs4v0kt9taoe3m5hd49dmrgun" if types.Format_Default == types.Format_DOLT_DEV { - headCommitHash = "r5hevva9fc9ul414fm5lo11r8vcqifc1" + headCommitHash = "sjrnn6go8q3gm8ikd91mel32u30kra8i" } return []SelectTest{ { diff --git a/go/store/cmd/noms/noms_root_test.go b/go/store/cmd/noms/noms_root_test.go index 49b6ce0694..4b98b20419 100644 --- a/go/store/cmd/noms/noms_root_test.go +++ b/go/store/cmd/noms/noms_root_test.go @@ -55,8 +55,8 @@ func (s *nomsRootTestSuite) TestBasic() { goldenHello := "u8g2r4qg97kkqn42lvao77st2mv3bpl0\n" goldenGoodbye := "70b9adi6amrab3a5t4hcibdob0cq49m0\n" if types.Format_Default == types.Format_DOLT_DEV { - goldenHello = "mmvr5g771a7eo56qpor2o0t5pvsolq75\n" - goldenGoodbye = "e77rueijm2122jim6ah09afjtqcpaagd\n" + goldenHello = "9n853u2sa9oebhernnq1kq2jghgk3oe1\n" + goldenGoodbye = "uspjpb3lfvutrns7crpvkfvor8md1ums\n" } ds, _ = datas.CommitValue(context.Background(), db, ds, types.String("hello!")) diff --git a/go/store/cmd/noms/noms_show.go b/go/store/cmd/noms/noms_show.go index c409b2aa56..e75d4ad4dc 100644 --- a/go/store/cmd/noms/noms_show.go +++ b/go/store/cmd/noms/noms_show.go @@ -34,6 +34,7 @@ import ( "github.com/dolthub/dolt/go/store/cmd/noms/util" "github.com/dolthub/dolt/go/store/config" "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/prolly/shim" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" @@ -127,7 +128,7 @@ func outputType(value types.Value) { var typeString string switch value := value.(type) { case types.SerialMessage: - switch serial.GetFileID(value) { + switch serial.GetFileID(value[message.MessagePrefixSz:]) { case serial.StoreRootFileID: typeString = "StoreRoot" case serial.TagFileID: @@ -159,14 +160,11 @@ func outputType(value types.Value) { func outputEncodedValue(ctx context.Context, w io.Writer, value types.Value) error { switch value := value.(type) { - case types.TupleRowStorage: - node := shim.NodeFromValue(value) - return tree.OutputProllyNode(w, node) // Some types of serial message need to be output here because of dependency cycles between types / tree package case types.SerialMessage: - switch serial.GetFileID(value) { + switch serial.GetFileID(value[message.MessagePrefixSz:]) { case serial.TableFileID: - msg := serial.GetRootAsTable(value, 0) + msg := serial.GetRootAsTable(value, message.MessagePrefixSz) fmt.Fprintf(w, "{\n") fmt.Fprintf(w, "\tSchema: #%s\n", hash.New(msg.SchemaBytes()).String()) @@ -193,10 +191,17 @@ func outputEncodedValue(ctx context.Context, w io.Writer, value types.Value) err return nil case serial.StoreRootFileID: - msg := serial.GetRootAsStoreRoot(value, 0) + msg := serial.GetRootAsStoreRoot(value, message.MessagePrefixSz) ambytes := msg.AddressMapBytes() node := tree.NodeFromBytes(ambytes) return tree.OutputAddressMapNode(w, node) + case serial.ProllyTreeNodeFileID: + fallthrough + case serial.AddressMapFileID: + fallthrough + case serial.CommitClosureFileID: + node := shim.NodeFromValue(value) + return tree.OutputProllyNode(w, node) default: return types.WriteEncodedValue(ctx, w, value) } diff --git a/go/store/datas/commit.go b/go/store/datas/commit.go index 7c526fac17..eff42125af 100644 --- a/go/store/datas/commit.go +++ b/go/store/datas/commit.go @@ -34,6 +34,7 @@ import ( "github.com/dolthub/dolt/go/store/d" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/nomdl" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" "github.com/dolthub/dolt/go/store/val" @@ -136,7 +137,7 @@ func NewCommitForValue(ctx context.Context, cs chunks.ChunkStore, vrw types.Valu return newCommitForValue(ctx, cs, vrw, ns, v, opts) } -func commit_flatbuffer(vaddr hash.Hash, opts CommitOptions, heights []uint64, parentsClosureAddr hash.Hash) ([]byte, uint64) { +func commit_flatbuffer(vaddr hash.Hash, opts CommitOptions, heights []uint64, parentsClosureAddr hash.Hash) (message.Message, uint64) { builder := flatbuffers.NewBuilder(1024) vaddroff := builder.CreateByteVector(vaddr[:]) @@ -174,8 +175,9 @@ func commit_flatbuffer(vaddr hash.Hash, opts CommitOptions, heights []uint64, pa serial.CommitAddDescription(builder, descoff) serial.CommitAddTimestampMillis(builder, opts.Meta.Timestamp) serial.CommitAddUserTimestampMillis(builder, opts.Meta.UserTimestamp) - builder.FinishWithFileIdentifier(serial.CommitEnd(builder), []byte(serial.CommitFileID)) - return builder.FinishedBytes(), maxheight + 1 + + bytes := message.FinishMessage(builder, serial.CommitEnd(builder), []byte(serial.CommitFileID)) + return bytes, maxheight + 1 } var commitKeyTupleDesc = val.NewTupleDescriptor( @@ -201,7 +203,7 @@ func newCommitForValue(ctx context.Context, cs chunks.ChunkStore, vrw types.Valu return nil, err } for i := range heights { - parents[i] = serial.GetRootAsCommit([]byte(parentValues[i].(types.SerialMessage)), 0) + parents[i] = serial.GetRootAsCommit([]byte(parentValues[i].(types.SerialMessage)), message.MessagePrefixSz) heights[i] = parents[i].Height() } parentClosureAddr, err := writeFbCommitParentClosure(ctx, cs, vrw, ns, parents, opts.Parents) @@ -261,7 +263,7 @@ func newCommitForValue(ctx context.Context, cs chunks.ChunkStore, vrw types.Valu func commitPtr(nbf *types.NomsBinFormat, v types.Value, r *types.Ref) (*Commit, error) { if nbf.UsesFlatbuffers() { bs := []byte(v.(types.SerialMessage)) - height := serial.GetRootAsCommit(bs, 0).Height() + height := serial.GetRootAsCommit(bs, message.MessagePrefixSz).Height() var addr hash.Hash if r != nil { addr = r.TargetHash() @@ -430,7 +432,7 @@ func FindClosureCommonAncestor(ctx context.Context, cl CommitClosure, cm *Commit func GetCommitParents(ctx context.Context, vr types.ValueReader, cv types.Value) ([]*Commit, error) { if sm, ok := cv.(types.SerialMessage); ok { data := []byte(sm) - if serial.GetFileID(data) != serial.CommitFileID { + if serial.GetFileID(data[message.MessagePrefixSz:]) != serial.CommitFileID { return nil, errors.New("GetCommitParents: provided value is not a commit.") } addrs, err := types.SerialCommitParentAddrs(vr.Format(), sm) @@ -443,7 +445,7 @@ func GetCommitParents(ctx context.Context, vr types.ValueReader, cv types.Value) if v == nil { return nil, fmt.Errorf("GetCommitParents: Did not find parent Commit in ValueReader: %s", addrs[i].String()) } - csm := serial.GetRootAsCommit([]byte(v.(types.SerialMessage)), 0) + csm := serial.GetRootAsCommit([]byte(v.(types.SerialMessage)), message.MessagePrefixSz) res[i] = &Commit{ val: v, height: csm.Height(), @@ -510,10 +512,10 @@ func GetCommitParents(ctx context.Context, vr types.ValueReader, cv types.Value) func GetCommitMeta(ctx context.Context, cv types.Value) (*CommitMeta, error) { if sm, ok := cv.(types.SerialMessage); ok { data := []byte(sm) - if serial.GetFileID(data) != serial.CommitFileID { + if serial.GetFileID(data[message.MessagePrefixSz:]) != serial.CommitFileID { return nil, errors.New("GetCommitMeta: provided value is not a commit.") } - cmsg := serial.GetRootAsCommit(data, 0) + cmsg := serial.GetRootAsCommit(data, message.MessagePrefixSz) ret := &CommitMeta{} ret.Name = string(cmsg.Name()) ret.Email = string(cmsg.Email()) @@ -546,10 +548,10 @@ func GetCommitMeta(ctx context.Context, cv types.Value) (*CommitMeta, error) { func GetCommittedValue(ctx context.Context, vr types.ValueReader, cv types.Value) (types.Value, error) { if sm, ok := cv.(types.SerialMessage); ok { data := []byte(sm) - if serial.GetFileID(data) != serial.CommitFileID { + if serial.GetFileID(data[message.MessagePrefixSz:]) != serial.CommitFileID { return nil, errors.New("GetCommittedValue: provided value is not a commit.") } - cmsg := serial.GetRootAsCommit(data, 0) + cmsg := serial.GetRootAsCommit(data, message.MessagePrefixSz) var roothash hash.Hash copy(roothash[:], cmsg.RootBytes()) return vr.ReadValue(ctx, roothash) @@ -672,7 +674,7 @@ func IsCommit(v types.Value) (bool, error) { return types.IsValueSubtypeOf(s.Format(), v, valueCommitType) } else if sm, ok := v.(types.SerialMessage); ok { data := []byte(sm) - return serial.GetFileID(data) == serial.CommitFileID, nil + return serial.GetFileID(data[message.MessagePrefixSz:]) == serial.CommitFileID, nil } else { return false, nil } diff --git a/go/store/datas/commit_closure.go b/go/store/datas/commit_closure.go index 5c6d4f7f9b..c5f5e3dc7f 100644 --- a/go/store/datas/commit_closure.go +++ b/go/store/datas/commit_closure.go @@ -24,6 +24,7 @@ import ( "github.com/dolthub/dolt/go/store/chunks" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" ) @@ -32,7 +33,7 @@ func newParentsClosureIterator(ctx context.Context, c *Commit, vr types.ValueRea sv := c.NomsValue() if _, ok := sv.(types.SerialMessage); ok { - msg := serial.GetRootAsCommit(sv.(types.SerialMessage), 0) + msg := serial.GetRootAsCommit(sv.(types.SerialMessage), message.MessagePrefixSz) addr := hash.New(msg.ParentClosureBytes()) if addr.IsEmpty() { return nil, nil @@ -44,7 +45,7 @@ func newParentsClosureIterator(ctx context.Context, c *Commit, vr types.ValueRea if types.IsNull(v) { return nil, fmt.Errorf("internal error or data loss: dangling commit parent closure for addr %s or commit %s", addr.String(), c.Addr().String()) } - node := tree.NodeFromBytes(v.(types.TupleRowStorage)) + node := tree.NodeFromBytes(v.(types.SerialMessage)) cc := prolly.NewCommitClosure(node, ns) ci, err := cc.IterAllReverse(ctx) if err != nil { @@ -391,7 +392,7 @@ func writeFbCommitParentClosure(ctx context.Context, cs chunks.ChunkStore, vrw t closures := make([]prolly.CommitClosure, len(parents)) for i := range addrs { if !types.IsNull(vs[i]) { - node := tree.NodeFromBytes(vs[i].(types.TupleRowStorage)) + node := tree.NodeFromBytes(vs[i].(types.SerialMessage)) closures[i] = prolly.NewCommitClosure(node, ns) } else { closures[i] = prolly.NewEmptyCommitClosure(ns) diff --git a/go/store/datas/commit_test.go b/go/store/datas/commit_test.go index f6867957fe..ffca43ae26 100644 --- a/go/store/datas/commit_test.go +++ b/go/store/datas/commit_test.go @@ -22,8 +22,10 @@ package datas import ( + "bytes" "context" "fmt" + "sort" "testing" "github.com/stretchr/testify/assert" @@ -421,6 +423,12 @@ func TestCommitParentsClosure(t *testing.T) { } assertCommitParentsClosure := func(v types.Value, es []expected) { + sort.Slice(es, func(i, j int) bool { + if es[i].height == es[j].height { + return bytes.Compare(es[i].hash[:], es[j].hash[:]) > 0 + } + return es[i].height > es[j].height + }) c, err := commitPtr(db.Format(), v, nil) if !assert.NoError(err) { return @@ -448,19 +456,14 @@ func TestCommitParentsClosure(t *testing.T) { assert.NoError(iter.Err()) } - // TODO: These tests rely on the hash values of the commits - // to assert the order of commits that are at the same height in the - // parent closure map. The values have been tweaked to currently pass - // with LD_1 and DOLT_DEV. - a, b, c, d := "ds-a", "ds-b", "ds-c", "ds-d" a1, a1a := addCommit(t, db, a, "a1") a2, a2a := addCommit(t, db, a, "a2", a1) - a3, a3a := addCommit(t, db, a, "a3 ", a2) + a3, a3a := addCommit(t, db, a, "a3", a2) b1, b1a := addCommit(t, db, b, "b1", a1) - b2, b2a := addCommit(t, db, b, "b2 ", b1) - b3, b3a := addCommit(t, db, b, "b3 ", b2) + b2, b2a := addCommit(t, db, b, "b2", b1) + b3, b3a := addCommit(t, db, b, "b3", b2) c1, c1a := addCommit(t, db, c, "c1", a3, b3) diff --git a/go/store/datas/dataset.go b/go/store/datas/dataset.go index d3336aa6ce..9636f71272 100644 --- a/go/store/datas/dataset.go +++ b/go/store/datas/dataset.go @@ -29,6 +29,7 @@ import ( "github.com/dolthub/dolt/go/gen/fb/serial" "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/types" ) @@ -148,7 +149,7 @@ type serialTagHead struct { } func newSerialTagHead(bs []byte, addr hash.Hash) serialTagHead { - return serialTagHead{serial.GetRootAsTag(bs, 0), addr} + return serialTagHead{serial.GetRootAsTag(bs, message.MessagePrefixSz), addr} } func (h serialTagHead) TypeName() string { @@ -185,7 +186,7 @@ type serialWorkingSetHead struct { } func newSerialWorkingSetHead(bs []byte, addr hash.Hash) serialWorkingSetHead { - return serialWorkingSetHead{serial.GetRootAsWorkingSet(bs, 0), addr} + return serialWorkingSetHead{serial.GetRootAsWorkingSet(bs, message.MessagePrefixSz), addr} } func (h serialWorkingSetHead) TypeName() string { @@ -304,13 +305,14 @@ func newHead(head types.Value, addr hash.Hash) (dsHead, error) { if sm, ok := head.(types.SerialMessage); ok { data := []byte(sm) - if serial.GetFileID(data) == serial.TagFileID { + fid := serial.GetFileID(data[message.MessagePrefixSz:]) + if fid == serial.TagFileID { return newSerialTagHead(data, addr), nil } - if serial.GetFileID(data) == serial.WorkingSetFileID { + if fid == serial.WorkingSetFileID { return newSerialWorkingSetHead(data, addr), nil } - if serial.GetFileID(data) == serial.CommitFileID { + if fid == serial.CommitFileID { return newSerialCommitHead(sm, addr), nil } } diff --git a/go/store/datas/refmap.go b/go/store/datas/refmap.go index 4799d4636e..760ac2a2c1 100644 --- a/go/store/datas/refmap.go +++ b/go/store/datas/refmap.go @@ -15,31 +15,29 @@ package datas import ( - "bytes" - flatbuffers "github.com/google/flatbuffers/go" "github.com/dolthub/dolt/go/gen/fb/serial" "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" ) -func storeroot_flatbuffer(am prolly.AddressMap) []byte { +func storeroot_flatbuffer(am prolly.AddressMap) message.Message { builder := flatbuffers.NewBuilder(1024) - ambytes := []byte(tree.ValueFromNode(am.Node()).(types.TupleRowStorage)) + ambytes := []byte(tree.ValueFromNode(am.Node()).(types.SerialMessage)) voff := builder.CreateByteVector(ambytes) serial.StoreRootStart(builder) serial.StoreRootAddAddressMap(builder, voff) - builder.FinishWithFileIdentifier(serial.StoreRootEnd(builder), []byte(serial.StoreRootFileID)) - return builder.FinishedBytes() + return message.FinishMessage(builder, serial.StoreRootEnd(builder), []byte(serial.StoreRootFileID)) } func parse_storeroot(bs []byte, ns tree.NodeStore) prolly.AddressMap { - if !bytes.Equal([]byte(serial.StoreRootFileID), bs[4:8]) { - panic("expected store root file id, got: " + string(bs[4:8])) + if serial.GetFileID(bs[message.MessagePrefixSz:]) != serial.StoreRootFileID { + panic("expected store root file id, got: " + serial.GetFileID(bs[message.MessagePrefixSz:])) } - sr := serial.GetRootAsStoreRoot(bs, 0) + sr := serial.GetRootAsStoreRoot(bs, message.MessagePrefixSz) mapbytes := sr.AddressMapBytes() node := tree.NodeFromBytes(mapbytes) return prolly.NewAddressMap(node, ns) diff --git a/go/store/datas/tag.go b/go/store/datas/tag.go index 1890963d2a..1cf0db75b8 100644 --- a/go/store/datas/tag.go +++ b/go/store/datas/tag.go @@ -23,6 +23,7 @@ import ( "github.com/dolthub/dolt/go/gen/fb/serial" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/nomdl" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/types" ) @@ -111,7 +112,7 @@ func newTag(ctx context.Context, db *database, commitAddr hash.Hash, meta *TagMe } } -func tag_flatbuffer(commitAddr hash.Hash, meta *TagMeta) []byte { +func tag_flatbuffer(commitAddr hash.Hash, meta *TagMeta) message.Message { builder := flatbuffers.NewBuilder(1024) addroff := builder.CreateByteVector(commitAddr[:]) var nameOff, emailOff, descOff flatbuffers.UOffsetT @@ -129,8 +130,7 @@ func tag_flatbuffer(commitAddr hash.Hash, meta *TagMeta) []byte { serial.TagAddTimestampMillis(builder, meta.Timestamp) serial.TagAddUserTimestampMillis(builder, meta.UserTimestamp) } - builder.FinishWithFileIdentifier(serial.TagEnd(builder), []byte(serial.TagFileID)) - return builder.FinishedBytes() + return message.FinishMessage(builder, serial.TagEnd(builder), []byte(serial.TagFileID)) } func IsTag(v types.Value) (bool, error) { @@ -138,7 +138,7 @@ func IsTag(v types.Value) (bool, error) { return types.IsValueSubtypeOf(s.Format(), v, valueTagType) } else if sm, ok := v.(types.SerialMessage); ok { data := []byte(sm) - return serial.GetFileID(data) == serial.TagFileID, nil + return serial.GetFileID(data[message.MessagePrefixSz:]) == serial.TagFileID, nil } return false, nil } diff --git a/go/store/datas/workingset.go b/go/store/datas/workingset.go index 1db83c023c..9c1c30e92e 100755 --- a/go/store/datas/workingset.go +++ b/go/store/datas/workingset.go @@ -21,6 +21,7 @@ import ( "github.com/dolthub/dolt/go/gen/fb/serial" "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly/message" "github.com/dolthub/dolt/go/store/types" ) @@ -148,7 +149,7 @@ func newWorkingSet(ctx context.Context, db *database, meta *WorkingSetMeta, work return ref.TargetHash(), ref, nil } -func workingset_flatbuffer(working hash.Hash, staged *hash.Hash, mergeState *MergeState, meta *WorkingSetMeta) []byte { +func workingset_flatbuffer(working hash.Hash, staged *hash.Hash, mergeState *MergeState, meta *WorkingSetMeta) message.Message { builder := flatbuffers.NewBuilder(1024) workingoff := builder.CreateByteVector(working[:]) var stagedOff, mergeStateOff flatbuffers.UOffsetT @@ -185,9 +186,7 @@ func workingset_flatbuffer(working hash.Hash, staged *hash.Hash, mergeState *Mer serial.WorkingSetAddDesc(builder, descOff) serial.WorkingSetAddTimestampMillis(builder, meta.Timestamp) } - builder.FinishWithFileIdentifier(serial.WorkingSetEnd(builder), []byte(serial.WorkingSetFileID)) - return builder.FinishedBytes() - + return message.FinishMessage(builder, serial.WorkingSetEnd(builder), []byte(serial.WorkingSetFileID)) } func NewMergeState(ctx context.Context, vrw types.ValueReadWriter, preMergeWorking types.Ref, commit *Commit) (*MergeState, error) { @@ -222,7 +221,7 @@ func IsWorkingSet(v types.Value) (bool, error) { // types.IsValueSubtypeOf is very strict about the type description. return s.Name() == workingSetName, nil } else if sm, ok := v.(types.SerialMessage); ok { - return serial.GetFileID([]byte(sm)) == serial.WorkingSetFileID, nil + return serial.GetFileID(sm[message.MessagePrefixSz:]) == serial.WorkingSetFileID, nil } else { return false, nil } diff --git a/go/store/prolly/commit_closure_test.go b/go/store/prolly/commit_closure_test.go index bdc09d998c..9264010cf4 100644 --- a/go/store/prolly/commit_closure_test.go +++ b/go/store/prolly/commit_closure_test.go @@ -148,7 +148,7 @@ func TestCommitClosure(t *testing.T) { assert.Equal(t, 4096, cc.Count()) // Walk the addresses in the root. - msg := message.Message(tree.ValueFromNode(cc.closure.root).(types.TupleRowStorage)) + msg := message.Message(tree.ValueFromNode(cc.closure.root).(types.SerialMessage)) numaddresses := 0 err = message.WalkAddresses(ctx, msg, func(ctx context.Context, addr hash.Hash) error { numaddresses++ diff --git a/go/store/prolly/message/address_map.go b/go/store/prolly/message/address_map.go index 0fa5e60505..cfc8884325 100644 --- a/go/store/prolly/message/address_map.go +++ b/go/store/prolly/message/address_map.go @@ -75,10 +75,10 @@ func (s AddressMapSerializer) Serialize(keys, addrs [][]byte, subtrees []uint64, } serial.AddressMapAddTreeLevel(b, uint8(level)) - return finishMessage(b, serial.AddressMapEnd(b), addressMapFileID) + return FinishMessage(b, serial.AddressMapEnd(b), addressMapFileID) } -func finishMessage(b *fb.Builder, off fb.UOffsetT, fileID []byte) []byte { +func FinishMessage(b *fb.Builder, off fb.UOffsetT, fileID []byte) Message { // We finish the buffer by prefixing it with: // 1) 1 byte NomsKind == TupleRowStorage. // 2) big endian uint16 representing the size of the message, not @@ -88,33 +88,33 @@ func finishMessage(b *fb.Builder, off fb.UOffsetT, fileID []byte) []byte { // codec. // // All accessors in this package expect this prefix to be on the front - // of the message bytes as well. See |messagePrefixSz|. + // of the message bytes as well. See |MessagePrefixSz|. - b.Prep(1, fb.SizeInt32+4+messagePrefixSz) + b.Prep(1, fb.SizeInt32+4+MessagePrefixSz) b.FinishWithFileIdentifier(off, fileID) - bytes := b.Bytes[b.Head()-messagePrefixSz:] + bytes := b.Bytes[b.Head()-MessagePrefixSz:] bytes[0] = byte(MessageTypesKind) binary.BigEndian.PutUint16(bytes[1:], uint16(len(b.Bytes)-int(b.Head()))) return bytes } func getAddressMapKeys(msg Message) (keys val.SlicedBuffer) { - am := serial.GetRootAsAddressMap(msg, messagePrefixSz) + am := serial.GetRootAsAddressMap(msg, MessagePrefixSz) keys.Buf = am.KeyItemsBytes() keys.Offs = getAddressMapKeyOffsets(am) return } func getAddressMapValues(msg Message) (values val.SlicedBuffer) { - am := serial.GetRootAsAddressMap(msg, messagePrefixSz) + am := serial.GetRootAsAddressMap(msg, MessagePrefixSz) values.Buf = am.AddressArrayBytes() values.Offs = offsetsForAddressArray(values.Buf) return } func walkAddressMapAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error { - am := serial.GetRootAsAddressMap(msg, messagePrefixSz) + am := serial.GetRootAsAddressMap(msg, MessagePrefixSz) arr := am.AddressArrayBytes() for i := 0; i < len(arr)/hash.ByteLen; i++ { addr := hash.New(arr[i*addrSize : (i+1)*addrSize]) @@ -126,7 +126,7 @@ func walkAddressMapAddresses(ctx context.Context, msg Message, cb func(ctx conte } func getAddressMapCount(msg Message) uint16 { - am := serial.GetRootAsAddressMap(msg, messagePrefixSz) + am := serial.GetRootAsAddressMap(msg, MessagePrefixSz) if am.KeyItemsLength() == 0 { return 0 } @@ -135,18 +135,18 @@ func getAddressMapCount(msg Message) uint16 { } func getAddressMapTreeLevel(msg Message) int { - am := serial.GetRootAsAddressMap(msg, messagePrefixSz) + am := serial.GetRootAsAddressMap(msg, MessagePrefixSz) return int(am.TreeLevel()) } func getAddressMapTreeCount(msg Message) int { - am := serial.GetRootAsAddressMap(msg, messagePrefixSz) + am := serial.GetRootAsAddressMap(msg, MessagePrefixSz) return int(am.TreeCount()) } func getAddressMapSubtrees(msg Message) []uint64 { counts := make([]uint64, getAddressMapCount(msg)) - am := serial.GetRootAsAddressMap(msg, messagePrefixSz) + am := serial.GetRootAsAddressMap(msg, MessagePrefixSz) return decodeVarints(am.SubtreeCountsBytes(), counts) } @@ -170,6 +170,6 @@ func estimateAddressMapSize(keys, addresses [][]byte, subtrees []uint64) (keySz, totalSz += len(subtrees) * binary.MaxVarintLen64 totalSz += 8 + 1 + 1 + 1 totalSz += 72 - totalSz += messagePrefixSz + totalSz += MessagePrefixSz return } diff --git a/go/store/prolly/message/commit_closure.go b/go/store/prolly/message/commit_closure.go index 090cf98a9c..dfb995e148 100644 --- a/go/store/prolly/message/commit_closure.go +++ b/go/store/prolly/message/commit_closure.go @@ -51,7 +51,7 @@ func offsetsForCommitClosureKeys(buf []byte) []byte { func getCommitClosureKeys(msg Message) val.SlicedBuffer { var ret val.SlicedBuffer - m := serial.GetRootAsCommitClosure(msg, messagePrefixSz) + m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz) ret.Buf = m.KeyItemsBytes() ret.Offs = offsetsForCommitClosureKeys(ret.Buf) return ret @@ -59,7 +59,7 @@ func getCommitClosureKeys(msg Message) val.SlicedBuffer { func getCommitClosureValues(msg Message) val.SlicedBuffer { var ret val.SlicedBuffer - m := serial.GetRootAsCommitClosure(msg, messagePrefixSz) + m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz) if m.AddressArrayLength() == 0 { ret.Buf = commitClosureEmptyValueBytes ret.Offs = commitClosureValueOffsets[:getCommitClosureCount(msg)*uint16Size] @@ -74,28 +74,28 @@ func getCommitClosureValues(msg Message) val.SlicedBuffer { const commitClosureKeyLength = 8 + 20 func getCommitClosureCount(msg Message) uint16 { - m := serial.GetRootAsCommitClosure(msg, messagePrefixSz) + m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz) return uint16(m.KeyItemsLength() / commitClosureKeyLength) } func getCommitClosureTreeLevel(msg Message) int { - m := serial.GetRootAsCommitClosure(msg, messagePrefixSz) + m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz) return int(m.TreeLevel()) } func getCommitClosureTreeCount(msg Message) int { - m := serial.GetRootAsCommitClosure(msg, messagePrefixSz) + m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz) return int(m.TreeCount()) } func getCommitClosureSubtrees(msg Message) []uint64 { counts := make([]uint64, getCommitClosureCount(msg)) - m := serial.GetRootAsCommitClosure(msg, messagePrefixSz) + m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz) return decodeVarints(m.SubtreeCountsBytes(), counts) } func walkCommitClosureAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error { - m := serial.GetRootAsCommitClosure(msg, messagePrefixSz) + m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz) arr := m.AddressArrayBytes() for i := 0; i < len(arr)/hash.ByteLen; i++ { addr := hash.New(arr[i*addrSize : (i+1)*addrSize]) @@ -153,7 +153,7 @@ func (s CommitClosureSerializer) Serialize(keys, addrs [][]byte, subtrees []uint } serial.CommitClosureAddTreeLevel(b, uint8(level)) - return finishMessage(b, serial.CommitClosureEnd(b), commitClosureFileID) + return FinishMessage(b, serial.CommitClosureEnd(b), commitClosureFileID) } func estimateCommitClosureSize(keys, addresses [][]byte, subtrees []uint64) (keySz, addrSz, totalSz int) { @@ -163,6 +163,6 @@ func estimateCommitClosureSize(keys, addresses [][]byte, subtrees []uint64) (key totalSz += len(subtrees) * binary.MaxVarintLen64 totalSz += 8 + 1 + 1 + 1 totalSz += 72 - totalSz += messagePrefixSz + totalSz += MessagePrefixSz return } diff --git a/go/store/prolly/message/message.go b/go/store/prolly/message/message.go index 228dc55f38..830e8f5560 100644 --- a/go/store/prolly/message/message.go +++ b/go/store/prolly/message/message.go @@ -24,9 +24,9 @@ import ( "github.com/dolthub/dolt/go/store/hash" ) -const MessageTypesKind int = 28 +const MessageTypesKind int = 27 -const messagePrefixSz = 3 +const MessagePrefixSz = 3 type Message []byte @@ -35,7 +35,7 @@ type Serializer interface { } func GetKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt uint16) { - id := serial.GetFileID(msg[messagePrefixSz:]) + id := serial.GetFileID(msg[MessagePrefixSz:]) if id == serial.ProllyTreeNodeFileID { return getProllyMapKeysAndValues(msg) @@ -57,7 +57,7 @@ func GetKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt uint16) { } func WalkAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error { - id := serial.GetFileID(msg[messagePrefixSz:]) + id := serial.GetFileID(msg[MessagePrefixSz:]) switch id { case serial.ProllyTreeNodeFileID: return walkProllyMapAddresses(ctx, msg, cb) @@ -71,7 +71,7 @@ func WalkAddresses(ctx context.Context, msg Message, cb func(ctx context.Context } func GetTreeLevel(msg Message) int { - id := serial.GetFileID(msg[messagePrefixSz:]) + id := serial.GetFileID(msg[MessagePrefixSz:]) switch id { case serial.ProllyTreeNodeFileID: return getProllyMapTreeLevel(msg) @@ -85,7 +85,7 @@ func GetTreeLevel(msg Message) int { } func GetTreeCount(msg Message) int { - id := serial.GetFileID(msg[messagePrefixSz:]) + id := serial.GetFileID(msg[MessagePrefixSz:]) switch id { case serial.ProllyTreeNodeFileID: return getProllyMapTreeCount(msg) @@ -99,7 +99,7 @@ func GetTreeCount(msg Message) int { } func GetSubtrees(msg Message) []uint64 { - id := serial.GetFileID(msg[messagePrefixSz:]) + id := serial.GetFileID(msg[MessagePrefixSz:]) switch id { case serial.ProllyTreeNodeFileID: return getProllyMapSubtrees(msg) diff --git a/go/store/prolly/message/prolly_map.go b/go/store/prolly/message/prolly_map.go index 07a9b76440..1703934cd9 100644 --- a/go/store/prolly/message/prolly_map.go +++ b/go/store/prolly/message/prolly_map.go @@ -93,11 +93,11 @@ func (s ProllyMapSerializer) Serialize(keys, values [][]byte, subtrees []uint64, serial.ProllyTreeNodeAddValueType(b, serial.ItemTypeTupleFormatAlpha) serial.ProllyTreeNodeAddTreeLevel(b, uint8(level)) - return finishMessage(b, serial.ProllyTreeNodeEnd(b), prollyMapFileID) + return FinishMessage(b, serial.ProllyTreeNodeEnd(b), prollyMapFileID) } func getProllyMapKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt uint16) { - pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz) + pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz) keys.Buf = pm.KeyItemsBytes() keys.Offs = getProllyMapKeyOffsets(pm) @@ -120,7 +120,7 @@ func getProllyMapKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt } func walkProllyMapAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error { - pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz) + pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz) arr := pm.AddressArrayBytes() for i := 0; i < len(arr)/hash.ByteLen; i++ { addr := hash.New(arr[i*addrSize : (i+1)*addrSize]) @@ -143,7 +143,7 @@ func walkProllyMapAddresses(ctx context.Context, msg Message, cb func(ctx contex } func getProllyMapCount(msg Message) uint16 { - pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz) + pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz) if pm.KeyItemsLength() == 0 { return 0 } @@ -152,18 +152,18 @@ func getProllyMapCount(msg Message) uint16 { } func getProllyMapTreeLevel(msg Message) int { - pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz) + pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz) return int(pm.TreeLevel()) } func getProllyMapTreeCount(msg Message) int { - pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz) + pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz) return int(pm.TreeCount()) } func getProllyMapSubtrees(msg Message) []uint64 { counts := make([]uint64, getProllyMapCount(msg)) - pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz) + pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz) return decodeVarints(pm.SubtreeCountsBytes(), counts) } @@ -213,7 +213,7 @@ func estimateProllyMapSize(keys, values [][]byte, subtrees []uint64, valAddrsCnt bufSz += 72 // vtable (approx) bufSz += 100 // padding? bufSz += valAddrsCnt * len(values) - bufSz += messagePrefixSz + bufSz += MessagePrefixSz return keySz, valSz, bufSz } diff --git a/go/store/prolly/shim/shim.go b/go/store/prolly/shim/shim.go index ab36d1c848..f4d3f8f418 100644 --- a/go/store/prolly/shim/shim.go +++ b/go/store/prolly/shim/shim.go @@ -25,7 +25,7 @@ import ( ) func NodeFromValue(v types.Value) tree.Node { - return tree.NodeFromBytes(v.(types.TupleRowStorage)) + return tree.NodeFromBytes(v.(types.SerialMessage)) } func ValueFromMap(m prolly.Map) types.Value { diff --git a/go/store/prolly/tree/content_address_test.go b/go/store/prolly/tree/content_address_test.go index eaf3dd922f..4d4c6c6294 100644 --- a/go/store/prolly/tree/content_address_test.go +++ b/go/store/prolly/tree/content_address_test.go @@ -27,11 +27,10 @@ import ( ) var goldenHash = hash.Hash{ - 0x39, 0x1c, 0xcb, 0xd8, - 0xea, 0xd2, 0xdc, 0x42, - 0x76, 0xb7, 0x38, 0xf1, - 0x0d, 0x4f, 0x48, 0x91, - 0x7d, 0x1f, 0xb7, 0xb4, + 0xde, 0x2c, 0x34, 0x6a, 0xb3, + 0x88, 0x6f, 0x95, 0xb, 0x3b, + 0x77, 0xa1, 0x1f, 0xfa, 0x4c, + 0xf1, 0xe8, 0xdb, 0xfa, 0xcc, } // todo(andy): need and analogous test in pkg prolly diff --git a/go/store/prolly/tree/node.go b/go/store/prolly/tree/node.go index c4f50f68e5..4a706e0554 100644 --- a/go/store/prolly/tree/node.go +++ b/go/store/prolly/tree/node.go @@ -235,5 +235,5 @@ func OutputAddressMapNode(w io.Writer, node Node) error { } func ValueFromNode(root Node) types.Value { - return types.TupleRowStorage(root.bytes()) + return types.SerialMessage(root.bytes()) } diff --git a/go/store/prolly/tree/node_test.go b/go/store/prolly/tree/node_test.go index 18cafa9645..54133e5ca8 100644 --- a/go/store/prolly/tree/node_test.go +++ b/go/store/prolly/tree/node_test.go @@ -104,7 +104,7 @@ func TestNodeDecodeValueCompatibility(t *testing.T) { v, err := vs.ReadValue(context.Background(), h) require.NoError(t, err) - assert.Equal(t, nd.bytes(), []byte(v.(types.TupleRowStorage))) + assert.Equal(t, nd.bytes(), []byte(v.(types.SerialMessage))) } func randomNodeItemPairs(t *testing.T, count int) (keys, values []Item) { diff --git a/go/store/types/codec.go b/go/store/types/codec.go index 703a2e0d01..0ae2b0ef57 100644 --- a/go/store/types/codec.go +++ b/go/store/types/codec.go @@ -325,7 +325,7 @@ func (b *binaryNomsReader) ReadInlineBlob() []byte { return bytes } -func (b *binaryNomsReader) readTupleRowStorage() []byte { +func (b *binaryNomsReader) readSerialMessage() []byte { size := uint32(b.readUint16()) // start at offset-3, to include the kind byte + Uint16 for size... bytes := b.buff[b.offset-3 : b.offset+size] diff --git a/go/store/types/noms_kind.go b/go/store/types/noms_kind.go index 79bbf3fb6f..aabc58093e 100644 --- a/go/store/types/noms_kind.go +++ b/go/store/types/noms_kind.go @@ -66,7 +66,6 @@ const ( PolygonKind SerialMessageKind - TupleRowStorageKind UnknownKind NomsKind = 255 ) @@ -99,7 +98,6 @@ func init() { KindToType[LineStringKind] = LineString{} KindToType[PolygonKind] = Polygon{} KindToType[SerialMessageKind] = SerialMessage{} - KindToType[TupleRowStorageKind] = TupleRowStorage{} SupportedKinds[BlobKind] = true SupportedKinds[BoolKind] = true @@ -128,45 +126,43 @@ func init() { SupportedKinds[LineStringKind] = true SupportedKinds[PolygonKind] = true SupportedKinds[SerialMessageKind] = true - SupportedKinds[TupleRowStorageKind] = true - if message.MessageTypesKind != int(TupleRowStorageKind) { - panic("internal error: message.MessageTypesKind != TupleRowStorageKind") + if message.MessageTypesKind != int(SerialMessageKind) { + panic("internal error: message.MessageTypesKind != SerialMessageKind") } } var KindToTypeSlice []Value var KindToString = map[NomsKind]string{ - UnknownKind: "unknown", - BlobKind: "Blob", - BoolKind: "Bool", - CycleKind: "Cycle", - ListKind: "List", - MapKind: "Map", - FloatKind: "Float", - RefKind: "Ref", - SetKind: "Set", - StructKind: "Struct", - StringKind: "String", - TypeKind: "Type", - UnionKind: "Union", - ValueKind: "Value", - UUIDKind: "UUID", - IntKind: "Int", - UintKind: "Uint", - NullKind: "Null", - TupleKind: "Tuple", - InlineBlobKind: "InlineBlob", - TimestampKind: "Timestamp", - DecimalKind: "Decimal", - JSONKind: "JSON", - GeometryKind: "Geometry", - PointKind: "Point", - LineStringKind: "LineString", - PolygonKind: "Polygon", - SerialMessageKind: "SerialMessage", - TupleRowStorageKind: "TupleRowStorage", + UnknownKind: "unknown", + BlobKind: "Blob", + BoolKind: "Bool", + CycleKind: "Cycle", + ListKind: "List", + MapKind: "Map", + FloatKind: "Float", + RefKind: "Ref", + SetKind: "Set", + StructKind: "Struct", + StringKind: "String", + TypeKind: "Type", + UnionKind: "Union", + ValueKind: "Value", + UUIDKind: "UUID", + IntKind: "Int", + UintKind: "Uint", + NullKind: "Null", + TupleKind: "Tuple", + InlineBlobKind: "InlineBlob", + TimestampKind: "Timestamp", + DecimalKind: "Decimal", + JSONKind: "JSON", + GeometryKind: "Geometry", + PointKind: "Point", + LineStringKind: "LineString", + PolygonKind: "Polygon", + SerialMessageKind: "SerialMessage", } // String returns the name of the kind. diff --git a/go/store/types/path.go b/go/store/types/path.go index 6fa351b52a..4250d2d172 100644 --- a/go/store/types/path.go +++ b/go/store/types/path.go @@ -34,6 +34,7 @@ import ( "github.com/dolthub/dolt/go/gen/fb/serial" "github.com/dolthub/dolt/go/store/d" "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly/message" ) // For an annotation like @type, 1st capture group is the annotation. @@ -235,9 +236,8 @@ func (fp FieldPath) Resolve(ctx context.Context, v Value, vr ValueReader) (Value return sv, nil } case SerialMessage: - data := []byte(v) - if serial.GetFileID(data) == serial.CommitFileID && fp.Name == "value" { - msg := serial.GetRootAsCommit(data, 0) + if serial.GetFileID(v[message.MessagePrefixSz:]) == serial.CommitFileID && fp.Name == "value" { + msg := serial.GetRootAsCommit(v, message.MessagePrefixSz) addr := hash.New(msg.RootBytes()) return vr.ReadValue(ctx, addr) } diff --git a/go/store/types/serial_message.go b/go/store/types/serial_message.go index 8a9d5b875d..f86a91fd48 100644 --- a/go/store/types/serial_message.go +++ b/go/store/types/serial_message.go @@ -17,13 +17,14 @@ package types import ( "bytes" "context" + "encoding/hex" "fmt" - "math" "strings" "time" "github.com/dolthub/dolt/go/gen/fb/serial" "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly/message" ) type SerialMessage []byte @@ -52,17 +53,18 @@ func (sm SerialMessage) Hash(nbf *NomsBinFormat) (hash.Hash, error) { } func (sm SerialMessage) HumanReadableString() string { - switch serial.GetFileID(sm) { + id := serial.GetFileID(sm[message.MessagePrefixSz:]) + switch id { case serial.StoreRootFileID: - msg := serial.GetRootAsStoreRoot([]byte(sm), 0) + msg := serial.GetRootAsStoreRoot([]byte(sm), message.MessagePrefixSz) ret := &strings.Builder{} mapbytes := msg.AddressMapBytes() - fmt.Fprintf(ret, "StoreRoot{%s}", TupleRowStorage(mapbytes).HumanReadableString()) + fmt.Fprintf(ret, "StoreRoot{%s}", SerialMessage(mapbytes).HumanReadableString()) return ret.String() case serial.TagFileID: return "Tag" case serial.WorkingSetFileID: - msg := serial.GetRootAsWorkingSet(sm, 0) + msg := serial.GetRootAsWorkingSet(sm, message.MessagePrefixSz) ret := &strings.Builder{} fmt.Fprintf(ret, "{\n") fmt.Fprintf(ret, "\tName: %s\n", msg.Name()) @@ -74,7 +76,7 @@ func (sm SerialMessage) HumanReadableString() string { fmt.Fprintf(ret, "}") return ret.String() case serial.CommitFileID: - msg := serial.GetRootAsCommit(sm, 0) + msg := serial.GetRootAsCommit(sm, message.MessagePrefixSz) ret := &strings.Builder{} fmt.Fprintf(ret, "{\n") fmt.Fprintf(ret, "\tName: %s\n", msg.Name()) @@ -102,18 +104,18 @@ func (sm SerialMessage) HumanReadableString() string { fmt.Fprintf(ret, "}") return ret.String() case serial.RootValueFileID: - msg := serial.GetRootAsRootValue(sm, 0) + msg := serial.GetRootAsRootValue(sm, message.MessagePrefixSz) ret := &strings.Builder{} fmt.Fprintf(ret, "{\n") fmt.Fprintf(ret, "\tFeatureVersion: %d\n", msg.FeatureVersion()) fmt.Fprintf(ret, "\tForeignKeys: #%s\n", hash.New(msg.ForeignKeyAddrBytes()).String()) fmt.Fprintf(ret, "\tSuperSchema: #%s\n", hash.New(msg.SuperSchemasAddrBytes()).String()) - fmt.Fprintf(ret, "\tTables: {\n\t%s", TupleRowStorage(msg.TablesBytes()).HumanReadableString()) + fmt.Fprintf(ret, "\tTables: {\n\t%s", SerialMessage(msg.TablesBytes()).HumanReadableString()) fmt.Fprintf(ret, "\t}\n") fmt.Fprintf(ret, "}") return ret.String() case serial.TableFileID: - msg := serial.GetRootAsTable(sm, 0) + msg := serial.GetRootAsTable(sm, message.MessagePrefixSz) ret := &strings.Builder{} fmt.Fprintf(ret, "{\n") @@ -126,16 +128,27 @@ func (sm SerialMessage) HumanReadableString() string { // TODO: can't use tree package to print here, creates a cycle fmt.Fprintf(ret, "\tPrimary index: prolly tree\n") - fmt.Fprintf(ret, "\tSecondary indexes: {\n\t%s\n", TupleRowStorage(msg.SecondaryIndexesBytes()).HumanReadableString()) + fmt.Fprintf(ret, "\tSecondary indexes: {\n\t%s\n", SerialMessage(msg.SecondaryIndexesBytes()).HumanReadableString()) fmt.Fprintf(ret, "\t}\n") fmt.Fprintf(ret, "}") return ret.String() - case serial.ProllyTreeNodeFileID: - return "ProllyTreeNode" case serial.AddressMapFileID: - return "AddressMap" + keys, values, cnt := message.GetKeysAndValues(message.Message(sm)) + var b strings.Builder + b.Write([]byte("AddressMap{\n")) + for i := uint16(0); i < cnt; i++ { + name := keys.GetSlice(int(i)) + addr := values.GetSlice(int(i)) + b.Write([]byte("\t")) + b.Write(name) + b.Write([]byte(": ")) + b.Write([]byte(hash.New(addr).String())) + b.Write([]byte("\n")) + } + b.Write([]byte("}")) + return b.String() default: - return "SerialMessage (HumanReadableString not implemented)" + return fmt.Sprintf("SerialMessage (HumanReadableString not implemented), [%v]: %s", id, strings.ToUpper(hex.EncodeToString(sm))) } } @@ -151,15 +164,15 @@ func (sm SerialMessage) Less(nbf *NomsBinFormat, other LesserValuable) (bool, er const SerialMessageRefHeight = 1024 func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { - switch serial.GetFileID([]byte(sm)) { + switch serial.GetFileID(sm[message.MessagePrefixSz:]) { case serial.StoreRootFileID: - msg := serial.GetRootAsStoreRoot([]byte(sm), 0) + msg := serial.GetRootAsStoreRoot([]byte(sm), message.MessagePrefixSz) if msg.AddressMapLength() > 0 { mapbytes := msg.AddressMapBytes() - return TupleRowStorage(mapbytes).walkRefs(nbf, cb) + return SerialMessage(mapbytes).walkRefs(nbf, cb) } case serial.TagFileID: - msg := serial.GetRootAsTag([]byte(sm), 0) + msg := serial.GetRootAsTag([]byte(sm), message.MessagePrefixSz) addr := hash.New(msg.CommitAddrBytes()) r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight) if err != nil { @@ -167,7 +180,7 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { } return cb(r) case serial.WorkingSetFileID: - msg := serial.GetRootAsWorkingSet([]byte(sm), 0) + msg := serial.GetRootAsWorkingSet([]byte(sm), message.MessagePrefixSz) addr := hash.New(msg.WorkingRootAddrBytes()) r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight) if err != nil { @@ -207,8 +220,8 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { } } case serial.RootValueFileID: - msg := serial.GetRootAsRootValue([]byte(sm), 0) - err := TupleRowStorage(msg.TablesBytes()).walkRefs(nbf, cb) + msg := serial.GetRootAsRootValue([]byte(sm), message.MessagePrefixSz) + err := SerialMessage(msg.TablesBytes()).walkRefs(nbf, cb) if err != nil { return err } @@ -233,7 +246,7 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { } } case serial.TableFileID: - msg := serial.GetRootAsTable([]byte(sm), 0) + msg := serial.GetRootAsTable([]byte(sm), message.MessagePrefixSz) addr := hash.New(msg.SchemaBytes()) r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight) if err != nil { @@ -300,7 +313,7 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { } } - err = TupleRowStorage(msg.SecondaryIndexesBytes()).walkRefs(nbf, cb) + err = SerialMessage(msg.SecondaryIndexesBytes()).walkRefs(nbf, cb) if err != nil { return err } @@ -315,7 +328,7 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { } return v.walkRefs(nbf, cb) } else { - return TupleRowStorage(mapbytes).walkRefs(nbf, cb) + return SerialMessage(mapbytes).walkRefs(nbf, cb) } case serial.CommitFileID: parents, err := SerialCommitParentAddrs(nbf, sm) @@ -331,7 +344,7 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { return err } } - msg := serial.GetRootAsCommit([]byte(sm), 0) + msg := serial.GetRootAsCommit([]byte(sm), message.MessagePrefixSz) addr := hash.New(msg.RootBytes()) r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight) if err != nil { @@ -355,14 +368,26 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { return nil case serial.ForeignKeyCollectionFileID: return nil + case serial.ProllyTreeNodeFileID: + fallthrough + case serial.AddressMapFileID: + fallthrough + case serial.CommitClosureFileID: + return message.WalkAddresses(context.TODO(), message.Message(sm), func(ctx context.Context, addr hash.Hash) error { + r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight) + if err != nil { + return err + } + return cb(r) + }) default: - return fmt.Errorf("unsupported SerialMessage message with FileID: %s", serial.GetFileID([]byte(sm))) + return fmt.Errorf("unsupported SerialMessage message with FileID: %s", serial.GetFileID(sm[message.MessagePrefixSz:])) } return nil } func SerialCommitParentAddrs(nbf *NomsBinFormat, sm SerialMessage) ([]hash.Hash, error) { - msg := serial.GetRootAsCommit([]byte(sm), 0) + msg := serial.GetRootAsCommit([]byte(sm), message.MessagePrefixSz) addrs := msg.ParentAddrsBytes() n := len(addrs) / 20 ret := make([]hash.Hash, n) @@ -375,7 +400,7 @@ func SerialCommitParentAddrs(nbf *NomsBinFormat, sm SerialMessage) ([]hash.Hash, } func (sm SerialMessage) readFrom(nbf *NomsBinFormat, b *binaryNomsReader) (Value, error) { - bytes := b.ReadInlineBlob() + bytes := b.readSerialMessage() return SerialMessage(bytes), nil } @@ -389,16 +414,6 @@ func (sm SerialMessage) typeOf() (*Type, error) { } func (sm SerialMessage) writeTo(w nomsWriter, nbf *NomsBinFormat) error { - byteLen := len(sm) - if byteLen > math.MaxUint16 { - return fmt.Errorf("SerialMessage has length %v when max is %v", byteLen, math.MaxUint16) - } - - err := SerialMessageKind.writeTo(w, nbf) - if err != nil { - return err - } - w.writeUint16(uint16(byteLen)) w.writeRaw(sm) return nil } diff --git a/go/store/types/tuplerowstorage.go b/go/store/types/tuplerowstorage.go deleted file mode 100755 index bec69d9a08..0000000000 --- a/go/store/types/tuplerowstorage.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "bytes" - "context" - "encoding/hex" - "strings" - - "github.com/dolthub/dolt/go/gen/fb/serial" - "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/dolt/go/store/prolly/message" -) - -// TupleRowStorage cribs its implementation from InlineBlob. It bridges -// prolly/message byte arrays between types.Value and prolly/tree.Node. -// -// Unlike SerialMessage, the byte array held in TupleRowStorage includes the -// NomsKind byte and the BigEndian uint16 size of the message. |writeTo| is -// simply a call through to writeRaw, and |readFrom| has to pick up bytes from -// the reader that have already been "read" to determine kind and size. - -type TupleRowStorage []byte - -func (v TupleRowStorage) Value(ctx context.Context) (Value, error) { - return v, nil -} - -func (v TupleRowStorage) Equals(other Value) bool { - v2, ok := other.(TupleRowStorage) - if !ok { - return false - } - - return bytes.Equal(v, v2) -} - -func (v TupleRowStorage) Less(nbf *NomsBinFormat, other LesserValuable) (bool, error) { - if v2, ok := other.(TupleRowStorage); ok { - return bytes.Compare(v, v2) == -1, nil - } - return TupleRowStorageKind < other.Kind(), nil -} - -func (v TupleRowStorage) Hash(nbf *NomsBinFormat) (hash.Hash, error) { - return getHash(v, nbf) -} - -func (v TupleRowStorage) isPrimitive() bool { - return true -} - -func (v TupleRowStorage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { - return message.WalkAddresses(context.TODO(), message.Message([]byte(v)), func(ctx context.Context, addr hash.Hash) error { - r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight) - if err != nil { - return err - } - return cb(r) - }) -} - -func (v TupleRowStorage) typeOf() (*Type, error) { - return PrimitiveTypeMap[TupleRowStorageKind], nil -} - -func (v TupleRowStorage) Kind() NomsKind { - return TupleRowStorageKind -} - -func (v TupleRowStorage) valueReadWriter() ValueReadWriter { - return nil -} - -func (v TupleRowStorage) writeTo(w nomsWriter, nbf *NomsBinFormat) error { - w.writeRaw(v) - return nil -} - -func (v TupleRowStorage) readFrom(nbf *NomsBinFormat, b *binaryNomsReader) (Value, error) { - bytes := b.readTupleRowStorage() - return TupleRowStorage(bytes), nil -} - -func (v TupleRowStorage) skip(nbf *NomsBinFormat, b *binaryNomsReader) { - size := uint32(b.readUint16()) - b.skipBytes(size) -} - -func (v TupleRowStorage) HumanReadableString() string { - if serial.GetFileID(v) == serial.AddressMapFileID { - keys, values, cnt := message.GetKeysAndValues(message.Message([]byte(v))) - var b strings.Builder - b.Write([]byte("AddressMap{\n")) - for i := uint16(0); i < cnt; i++ { - name := keys.GetSlice(int(i)) - addr := values.GetSlice(int(i)) - b.Write([]byte("\t")) - b.Write(name) - b.Write([]byte(": ")) - b.Write([]byte(hash.New(addr).String())) - b.Write([]byte("\n")) - } - b.Write([]byte("}")) - return b.String() - } else { - return strings.ToUpper(hex.EncodeToString(v)) - } -}