mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-03-20 17:20:43 -05:00
Code Issues Packages Projects Releases 149 Wiki Activity

go/store/types: SerialMessage: Unify TupleRowStorage and SerialMessage.

Browse Source 
Get rid of TupleRowStorage and use SerialMessage everywhere. In turn, change
SerialMessage so that it carries its kind and size prefix as part of itself.
This commit is contained in:
Aaron Son
2022-07-22 11:47:36 -07:00
parent 8d0d368f29
commit 4f3f4e17af
31 changed files with 233 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
go/cmd/dolt/commands/roots.go
View File

@@ -34,6 +34,7 @@ import (
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/nbs"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/spec"
"github.com/dolthub/dolt/go/store/types"
@@ -151,8 +152,8 @@ func (cmd RootsCmd) processTableFile(ctx context.Context, path string, modified
cli.Println()
}
} else if sm, ok := value.(types.SerialMessage); ok {
if serial.GetFileID([]byte(sm)) == serial.StoreRootFileID {
msg := serial.GetRootAsStoreRoot([]byte(sm), 0)
if serial.GetFileID(sm[message.MessagePrefixSz:]) == serial.StoreRootFileID {
msg := serial.GetRootAsStoreRoot([]byte(sm), message.MessagePrefixSz)
ambytes := msg.AddressMapBytes()
node := tree.NodeFromBytes(ambytes)
err := tree.OutputAddressMapNode(cli.OutStream, node)

2
go/libraries/doltcore/doltdb/durable/index.go
View File

@@ -250,7 +250,7 @@ func (i prollyIndex) Format() *types.NomsBinFormat {
// bytes implements Index.
func (i prollyIndex) bytes() ([]byte, error) {
return []byte(shim.ValueFromMap(i.index).(types.TupleRowStorage)), nil
return []byte(shim.ValueFromMap(i.index).(types.SerialMessage)), nil
}
var _ Index = prollyIndex{}

16
go/libraries/doltcore/doltdb/durable/table.go
View File

@@ -30,6 +30,7 @@ import (
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/pool"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
@@ -194,11 +195,12 @@ func TableFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
err = errors.New("table ref is unexpected noms value; not SerialMessage")
return nil, err
}
if serial.GetFileID([]byte(sm)) != serial.TableFileID {
err = errors.New("table ref is unexpected noms value; GetFileID == " + serial.GetFileID([]byte(sm)))
id := serial.GetFileID(sm[message.MessagePrefixSz:])
if id != serial.TableFileID {
err = errors.New("table ref is unexpected noms value; GetFileID == " + id)
return nil, err
}
return doltDevTable{vrw, ns, serial.GetRootAsTable([]byte(sm), 0)}, nil
return doltDevTable{vrw, ns, serial.GetRootAsTable([]byte(sm), message.MessagePrefixSz)}, nil
}
}
@@ -745,7 +747,7 @@ func (fields serialTableFields) write() *serial.Table {
builder := flatbuffers.NewBuilder(1024)
indexesam := fields.indexes
indexesbytes := []byte(tree.ValueFromNode(indexesam.Node()).(types.TupleRowStorage))
indexesbytes := []byte(tree.ValueFromNode(indexesam.Node()).(types.SerialMessage))
schemaoff := builder.CreateByteVector(fields.schema)
rowsoff := builder.CreateByteVector(fields.rows)
@@ -772,8 +774,8 @@ func (fields serialTableFields) write() *serial.Table {
serial.TableAddConflicts(builder, conflictsoff)
serial.TableAddViolations(builder, violationsoff)
serial.TableAddArtifacts(builder, artifactsoff)
builder.FinishWithFileIdentifier(serial.TableEnd(builder), []byte(serial.TableFileID))
return serial.GetRootAsTable(builder.FinishedBytes(), 0)
bs := message.FinishMessage(builder, serial.TableEnd(builder), []byte(serial.TableFileID))
return serial.GetRootAsTable(bs, message.MessagePrefixSz)
}
func newDoltDevTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows Index, indexes IndexSet, autoIncVal types.Value) (Table, error) {
@@ -871,7 +873,7 @@ func (t doltDevTable) GetTableRows(ctx context.Context) (Index, error) {
if err != nil {
return nil, err
}
m := shim.MapFromValue(types.TupleRowStorage(rowbytes), sch, t.ns)
m := shim.MapFromValue(types.SerialMessage(rowbytes), sch, t.ns)
return IndexFromProllyMap(m), nil
}
}

12
go/libraries/doltcore/doltdb/foreign_key_serialization.go
View File

@@ -22,6 +22,7 @@ import (
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/marshal"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/types"
)
@@ -84,11 +85,11 @@ func serializeNomsForeignKeys(ctx context.Context, vrw types.ValueReadWriter, fk
// deserializeNomsForeignKeys returns a new ForeignKeyCollection using the provided map returned previously by GetMap.
func deserializeFlatbufferForeignKeys(msg types.SerialMessage) (*ForeignKeyCollection, error) {
if serial.GetFileID(msg) != serial.ForeignKeyCollectionFileID {
if serial.GetFileID(msg[message.MessagePrefixSz:]) != serial.ForeignKeyCollectionFileID {
return nil, fmt.Errorf("expect Serial Message with ForeignKeyCollectionFileID")
}
c := serial.GetRootAsForeignKeyCollection(msg, 0)
c := serial.GetRootAsForeignKeyCollection(msg, message.MessagePrefixSz)
collection := &ForeignKeyCollection{
foreignKeys: make(map[string]ForeignKey, c.ForeignKeysLength()),
}
@@ -203,8 +204,7 @@ func serializeFlatbufferForeignKeys(fkc *ForeignKeyCollection) types.SerialMessa
serial.ForeignKeyCollectionStart(b)
serial.ForeignKeyCollectionAddForeignKeys(b, vec)
o := serial.ForeignKeyCollectionEnd(b)
b.FinishWithFileIdentifier(o, []byte(serial.ForeignKeyCollectionFileID))
return types.SerialMessage(b.FinishedBytes())
return []byte(message.FinishMessage(b, o, []byte(serial.ForeignKeyCollectionFileID)))
}
func serializeStringVector(b *fb.Builder, s []string) fb.UOffsetT {
@@ -228,9 +228,9 @@ func serializeUint64Vector(b *fb.Builder, u []uint64) fb.UOffsetT {
}
func emptyForeignKeyCollection(msg types.SerialMessage) bool {
if serial.GetFileID(msg) != serial.ForeignKeyCollectionFileID {
if serial.GetFileID(msg[message.MessagePrefixSz:]) != serial.ForeignKeyCollectionFileID {
return false
}
c := serial.GetRootAsForeignKeyCollection(msg, 0)
c := serial.GetRootAsForeignKeyCollection(msg, message.MessagePrefixSz)
return c.ForeignKeysLength() == 0
}

20
go/libraries/doltcore/doltdb/root_val.go
View File

@@ -29,6 +29,7 @@ import (
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
@@ -255,7 +256,7 @@ func newRootValue(vrw types.ValueReadWriter, ns tree.NodeStore, v types.Value) (
var storage rvStorage
if vrw.Format().UsesFlatbuffers() {
srv := serial.GetRootAsRootValue([]byte(v.(types.SerialMessage)), 0)
srv := serial.GetRootAsRootValue([]byte(v.(types.SerialMessage)), message.MessagePrefixSz)
storage = fbRvStorage{srv}
} else {
st, ok := v.(types.Struct)
@@ -306,7 +307,7 @@ func decodeRootNomsValue(vrw types.ValueReadWriter, ns tree.NodeStore, val types
func isRootValue(nbf *types.NomsBinFormat, val types.Value) bool {
if nbf.UsesFlatbuffers() {
if sm, ok := val.(types.SerialMessage); ok {
return string(serial.GetFileID([]byte(sm))) == serial.RootValueFileID
return string(serial.GetFileID(sm[message.MessagePrefixSz:])) == serial.RootValueFileID
}
} else {
if st, ok := val.(types.Struct); ok {
@@ -321,7 +322,7 @@ func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter, ns tree.Node
builder := flatbuffers.NewBuilder(80)
emptyam := prolly.NewEmptyAddressMap(ns)
ambytes := []byte(tree.ValueFromNode(emptyam.Node()).(types.TupleRowStorage))
ambytes := []byte(tree.ValueFromNode(emptyam.Node()).(types.SerialMessage))
tablesoff := builder.CreateByteVector(ambytes)
var empty hash.Hash
@@ -332,8 +333,7 @@ func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter, ns tree.Node
serial.RootValueAddTables(builder, tablesoff)
serial.RootValueAddForeignKeyAddr(builder, fkoff)
serial.RootValueAddSuperSchemasAddr(builder, ssoff)
builder.FinishWithFileIdentifier(serial.RootValueEnd(builder), []byte(serial.RootValueFileID))
bs := builder.FinishedBytes()
bs := message.FinishMessage(builder, serial.RootValueEnd(builder), []byte(serial.RootValueFileID))
return newRootValue(vrw, ns, types.SerialMessage(bs))
}
@@ -1143,7 +1143,7 @@ func (r fbRvStorage) GetFeatureVersion() (FeatureVersion, bool, error) {
func (r fbRvStorage) getAddressMap(vrw types.ValueReadWriter, ns tree.NodeStore) prolly.AddressMap {
tbytes := r.srv.TablesBytes()
node := shim.NodeFromValue(types.TupleRowStorage(tbytes))
node := shim.NodeFromValue(types.SerialMessage(tbytes))
return prolly.NewAddressMap(node, ns)
}
@@ -1258,7 +1258,7 @@ func (r fbRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWrite
return nil, err
}
ambytes := []byte(tree.ValueFromNode(am.Node()).(types.TupleRowStorage))
ambytes := []byte(tree.ValueFromNode(am.Node()).(types.SerialMessage))
tablesoff := builder.CreateByteVector(ambytes)
fkoff := builder.CreateByteVector(r.srv.ForeignKeyAddrBytes())
@@ -1268,9 +1268,9 @@ func (r fbRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWrite
serial.RootValueAddTables(builder, tablesoff)
serial.RootValueAddForeignKeyAddr(builder, fkoff)
serial.RootValueAddSuperSchemasAddr(builder, ssoff)
builder.FinishWithFileIdentifier(serial.RootValueEnd(builder), []byte(serial.RootValueFileID))
bs := builder.FinishedBytes()
return fbRvStorage{serial.GetRootAsRootValue(bs, 0)}, nil
bs := message.FinishMessage(builder, serial.RootValueEnd(builder), []byte(serial.RootValueFileID))
return fbRvStorage{serial.GetRootAsRootValue(bs, message.MessagePrefixSz)}, nil
}
func (r fbRvStorage) SetForeignKeyMap(ctx context.Context, vrw types.ValueReadWriter, v types.Value) (rvStorage, error) {

9
go/libraries/doltcore/schema/encoding/serialization.go
View File

@@ -25,6 +25,7 @@ import (
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/types"
)
@@ -62,8 +63,8 @@ func serializeSchemaAsFlatbuffer(sch schema.Schema) ([]byte, error) {
serial.TableSchemaAddSecondaryIndexes(b, indexes)
serial.TableSchemaAddChecks(b, checks)
root := serial.TableSchemaEnd(b)
b.FinishWithFileIdentifier(root, []byte(serial.TableSchemaFileID))
return b.FinishedBytes(), nil
bs := message.FinishMessage(b, root, []byte(serial.TableSchemaFileID))
return bs, nil
}
// DeserializeSchema deserializes a schema.Schema from a serial.Message.
@@ -75,8 +76,8 @@ func DeserializeSchema(ctx context.Context, nbf *types.NomsBinFormat, v types.Va
}
func deserializeSchemaFromFlatbuffer(ctx context.Context, buf []byte) (schema.Schema, error) {
assertTrue(serial.GetFileID(buf) == serial.TableSchemaFileID)
s := serial.GetRootAsTableSchema(buf, 0)
assertTrue(serial.GetFileID(buf[message.MessagePrefixSz:]) == serial.TableSchemaFileID)
s := serial.GetRootAsTableSchema(buf, message.MessagePrefixSz)
cols, err := deserializeColumns(ctx, s)
if err != nil {

2
go/libraries/doltcore/schema/typeinfo/typeconverter.go
View File

@@ -131,7 +131,7 @@ func wrapConvertValueToNomsValue(
vInt = float64(val)
case types.InlineBlob:
vInt = *(*string)(unsafe.Pointer(&val))
case types.TupleRowStorage:
case types.SerialMessage:
vInt = *(*string)(unsafe.Pointer(&val))
case types.Int:
vInt = int64(val)

2
go/libraries/doltcore/sqle/sqlselect_test.go
View File

@@ -87,7 +87,7 @@ func LoadedLocalLocation() *time.Location {
func BasicSelectTests() []SelectTest {
headCommitHash := "73hc2robs4v0kt9taoe3m5hd49dmrgun"
if types.Format_Default == types.Format_DOLT_DEV {
headCommitHash = "r5hevva9fc9ul414fm5lo11r8vcqifc1"
headCommitHash = "sjrnn6go8q3gm8ikd91mel32u30kra8i"
}
return []SelectTest{
{

4
go/store/cmd/noms/noms_root_test.go
View File

@@ -55,8 +55,8 @@ func (s *nomsRootTestSuite) TestBasic() {
goldenHello := "u8g2r4qg97kkqn42lvao77st2mv3bpl0\n"
goldenGoodbye := "70b9adi6amrab3a5t4hcibdob0cq49m0\n"
if types.Format_Default == types.Format_DOLT_DEV {
goldenHello = "mmvr5g771a7eo56qpor2o0t5pvsolq75\n"
goldenGoodbye = "e77rueijm2122jim6ah09afjtqcpaagd\n"
goldenHello = "9n853u2sa9oebhernnq1kq2jghgk3oe1\n"
goldenGoodbye = "uspjpb3lfvutrns7crpvkfvor8md1ums\n"
}
ds, _ = datas.CommitValue(context.Background(), db, ds, types.String("hello!"))

19
go/store/cmd/noms/noms_show.go
View File

@@ -34,6 +34,7 @@ import (
"github.com/dolthub/dolt/go/store/cmd/noms/util"
"github.com/dolthub/dolt/go/store/config"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
@@ -127,7 +128,7 @@ func outputType(value types.Value) {
var typeString string
switch value := value.(type) {
case types.SerialMessage:
switch serial.GetFileID(value) {
switch serial.GetFileID(value[message.MessagePrefixSz:]) {
case serial.StoreRootFileID:
typeString = "StoreRoot"
case serial.TagFileID:
@@ -159,14 +160,11 @@ func outputType(value types.Value) {
func outputEncodedValue(ctx context.Context, w io.Writer, value types.Value) error {
switch value := value.(type) {
case types.TupleRowStorage:
node := shim.NodeFromValue(value)
return tree.OutputProllyNode(w, node)
// Some types of serial message need to be output here because of dependency cycles between types / tree package
case types.SerialMessage:
switch serial.GetFileID(value) {
switch serial.GetFileID(value[message.MessagePrefixSz:]) {
case serial.TableFileID:
msg := serial.GetRootAsTable(value, 0)
msg := serial.GetRootAsTable(value, message.MessagePrefixSz)
fmt.Fprintf(w, "{\n")
fmt.Fprintf(w, "\tSchema: #%s\n", hash.New(msg.SchemaBytes()).String())
@@ -193,10 +191,17 @@ func outputEncodedValue(ctx context.Context, w io.Writer, value types.Value) err
return nil
case serial.StoreRootFileID:
msg := serial.GetRootAsStoreRoot(value, 0)
msg := serial.GetRootAsStoreRoot(value, message.MessagePrefixSz)
ambytes := msg.AddressMapBytes()
node := tree.NodeFromBytes(ambytes)
return tree.OutputAddressMapNode(w, node)
case serial.ProllyTreeNodeFileID:
fallthrough
case serial.AddressMapFileID:
fallthrough
case serial.CommitClosureFileID:
node := shim.NodeFromValue(value)
return tree.OutputProllyNode(w, node)
default:
return types.WriteEncodedValue(ctx, w, value)
}

26
go/store/datas/commit.go
View File

@@ -34,6 +34,7 @@ import (
"github.com/dolthub/dolt/go/store/d"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/nomdl"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
@@ -136,7 +137,7 @@ func NewCommitForValue(ctx context.Context, cs chunks.ChunkStore, vrw types.Valu
return newCommitForValue(ctx, cs, vrw, ns, v, opts)
}
func commit_flatbuffer(vaddr hash.Hash, opts CommitOptions, heights []uint64, parentsClosureAddr hash.Hash) ([]byte, uint64) {
func commit_flatbuffer(vaddr hash.Hash, opts CommitOptions, heights []uint64, parentsClosureAddr hash.Hash) (message.Message, uint64) {
builder := flatbuffers.NewBuilder(1024)
vaddroff := builder.CreateByteVector(vaddr[:])
@@ -174,8 +175,9 @@ func commit_flatbuffer(vaddr hash.Hash, opts CommitOptions, heights []uint64, pa
serial.CommitAddDescription(builder, descoff)
serial.CommitAddTimestampMillis(builder, opts.Meta.Timestamp)
serial.CommitAddUserTimestampMillis(builder, opts.Meta.UserTimestamp)
builder.FinishWithFileIdentifier(serial.CommitEnd(builder), []byte(serial.CommitFileID))
return builder.FinishedBytes(), maxheight + 1
bytes := message.FinishMessage(builder, serial.CommitEnd(builder), []byte(serial.CommitFileID))
return bytes, maxheight + 1
}
var commitKeyTupleDesc = val.NewTupleDescriptor(
@@ -201,7 +203,7 @@ func newCommitForValue(ctx context.Context, cs chunks.ChunkStore, vrw types.Valu
return nil, err
}
for i := range heights {
parents[i] = serial.GetRootAsCommit([]byte(parentValues[i].(types.SerialMessage)), 0)
parents[i] = serial.GetRootAsCommit([]byte(parentValues[i].(types.SerialMessage)), message.MessagePrefixSz)
heights[i] = parents[i].Height()
}
parentClosureAddr, err := writeFbCommitParentClosure(ctx, cs, vrw, ns, parents, opts.Parents)
@@ -261,7 +263,7 @@ func newCommitForValue(ctx context.Context, cs chunks.ChunkStore, vrw types.Valu
func commitPtr(nbf *types.NomsBinFormat, v types.Value, r *types.Ref) (*Commit, error) {
if nbf.UsesFlatbuffers() {
bs := []byte(v.(types.SerialMessage))
height := serial.GetRootAsCommit(bs, 0).Height()
height := serial.GetRootAsCommit(bs, message.MessagePrefixSz).Height()
var addr hash.Hash
if r != nil {
addr = r.TargetHash()
@@ -430,7 +432,7 @@ func FindClosureCommonAncestor(ctx context.Context, cl CommitClosure, cm *Commit
func GetCommitParents(ctx context.Context, vr types.ValueReader, cv types.Value) ([]*Commit, error) {
if sm, ok := cv.(types.SerialMessage); ok {
data := []byte(sm)
if serial.GetFileID(data) != serial.CommitFileID {
if serial.GetFileID(data[message.MessagePrefixSz:]) != serial.CommitFileID {
return nil, errors.New("GetCommitParents: provided value is not a commit.")
}
addrs, err := types.SerialCommitParentAddrs(vr.Format(), sm)
@@ -443,7 +445,7 @@ func GetCommitParents(ctx context.Context, vr types.ValueReader, cv types.Value)
if v == nil {
return nil, fmt.Errorf("GetCommitParents: Did not find parent Commit in ValueReader: %s", addrs[i].String())
}
csm := serial.GetRootAsCommit([]byte(v.(types.SerialMessage)), 0)
csm := serial.GetRootAsCommit([]byte(v.(types.SerialMessage)), message.MessagePrefixSz)
res[i] = &Commit{
val: v,
height: csm.Height(),
@@ -510,10 +512,10 @@ func GetCommitParents(ctx context.Context, vr types.ValueReader, cv types.Value)
func GetCommitMeta(ctx context.Context, cv types.Value) (*CommitMeta, error) {
if sm, ok := cv.(types.SerialMessage); ok {
data := []byte(sm)
if serial.GetFileID(data) != serial.CommitFileID {
if serial.GetFileID(data[message.MessagePrefixSz:]) != serial.CommitFileID {
return nil, errors.New("GetCommitMeta: provided value is not a commit.")
}
cmsg := serial.GetRootAsCommit(data, 0)
cmsg := serial.GetRootAsCommit(data, message.MessagePrefixSz)
ret := &CommitMeta{}
ret.Name = string(cmsg.Name())
ret.Email = string(cmsg.Email())
@@ -546,10 +548,10 @@ func GetCommitMeta(ctx context.Context, cv types.Value) (*CommitMeta, error) {
func GetCommittedValue(ctx context.Context, vr types.ValueReader, cv types.Value) (types.Value, error) {
if sm, ok := cv.(types.SerialMessage); ok {
data := []byte(sm)
if serial.GetFileID(data) != serial.CommitFileID {
if serial.GetFileID(data[message.MessagePrefixSz:]) != serial.CommitFileID {
return nil, errors.New("GetCommittedValue: provided value is not a commit.")
}
cmsg := serial.GetRootAsCommit(data, 0)
cmsg := serial.GetRootAsCommit(data, message.MessagePrefixSz)
var roothash hash.Hash
copy(roothash[:], cmsg.RootBytes())
return vr.ReadValue(ctx, roothash)
@@ -672,7 +674,7 @@ func IsCommit(v types.Value) (bool, error) {
return types.IsValueSubtypeOf(s.Format(), v, valueCommitType)
} else if sm, ok := v.(types.SerialMessage); ok {
data := []byte(sm)
return serial.GetFileID(data) == serial.CommitFileID, nil
return serial.GetFileID(data[message.MessagePrefixSz:]) == serial.CommitFileID, nil
} else {
return false, nil
}

7
go/store/datas/commit_closure.go
View File

@@ -24,6 +24,7 @@ import (
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -32,7 +33,7 @@ func newParentsClosureIterator(ctx context.Context, c *Commit, vr types.ValueRea
sv := c.NomsValue()
if _, ok := sv.(types.SerialMessage); ok {
msg := serial.GetRootAsCommit(sv.(types.SerialMessage), 0)
msg := serial.GetRootAsCommit(sv.(types.SerialMessage), message.MessagePrefixSz)
addr := hash.New(msg.ParentClosureBytes())
if addr.IsEmpty() {
return nil, nil
@@ -44,7 +45,7 @@ func newParentsClosureIterator(ctx context.Context, c *Commit, vr types.ValueRea
if types.IsNull(v) {
return nil, fmt.Errorf("internal error or data loss: dangling commit parent closure for addr %s or commit %s", addr.String(), c.Addr().String())
}
node := tree.NodeFromBytes(v.(types.TupleRowStorage))
node := tree.NodeFromBytes(v.(types.SerialMessage))
cc := prolly.NewCommitClosure(node, ns)
ci, err := cc.IterAllReverse(ctx)
if err != nil {
@@ -391,7 +392,7 @@ func writeFbCommitParentClosure(ctx context.Context, cs chunks.ChunkStore, vrw t
closures := make([]prolly.CommitClosure, len(parents))
for i := range addrs {
if !types.IsNull(vs[i]) {
node := tree.NodeFromBytes(vs[i].(types.TupleRowStorage))
node := tree.NodeFromBytes(vs[i].(types.SerialMessage))
closures[i] = prolly.NewCommitClosure(node, ns)
} else {
closures[i] = prolly.NewEmptyCommitClosure(ns)

19
go/store/datas/commit_test.go
View File

@@ -22,8 +22,10 @@
package datas
import (
"bytes"
"context"
"fmt"
"sort"
"testing"
"github.com/stretchr/testify/assert"
@@ -421,6 +423,12 @@ func TestCommitParentsClosure(t *testing.T) {
}
assertCommitParentsClosure := func(v types.Value, es []expected) {
sort.Slice(es, func(i, j int) bool {
if es[i].height == es[j].height {
return bytes.Compare(es[i].hash[:], es[j].hash[:]) > 0
}
return es[i].height > es[j].height
})
c, err := commitPtr(db.Format(), v, nil)
if !assert.NoError(err) {
return
@@ -448,19 +456,14 @@ func TestCommitParentsClosure(t *testing.T) {
assert.NoError(iter.Err())
}
// TODO: These tests rely on the hash values of the commits
// to assert the order of commits that are at the same height in the
// parent closure map. The values have been tweaked to currently pass
// with LD_1 and DOLT_DEV.
a, b, c, d := "ds-a", "ds-b", "ds-c", "ds-d"
a1, a1a := addCommit(t, db, a, "a1")
a2, a2a := addCommit(t, db, a, "a2", a1)
a3, a3a := addCommit(t, db, a, "a3 ", a2)
a3, a3a := addCommit(t, db, a, "a3", a2)
b1, b1a := addCommit(t, db, b, "b1", a1)
b2, b2a := addCommit(t, db, b, "b2 ", b1)
b3, b3a := addCommit(t, db, b, "b3 ", b2)
b2, b2a := addCommit(t, db, b, "b2", b1)
b3, b3a := addCommit(t, db, b, "b3", b2)
c1, c1a := addCommit(t, db, c, "c1", a3, b3)

12
go/store/datas/dataset.go
View File

@@ -29,6 +29,7 @@ import (
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/types"
)
@@ -148,7 +149,7 @@ type serialTagHead struct {
}
func newSerialTagHead(bs []byte, addr hash.Hash) serialTagHead {
return serialTagHead{serial.GetRootAsTag(bs, 0), addr}
return serialTagHead{serial.GetRootAsTag(bs, message.MessagePrefixSz), addr}
}
func (h serialTagHead) TypeName() string {
@@ -185,7 +186,7 @@ type serialWorkingSetHead struct {
}
func newSerialWorkingSetHead(bs []byte, addr hash.Hash) serialWorkingSetHead {
return serialWorkingSetHead{serial.GetRootAsWorkingSet(bs, 0), addr}
return serialWorkingSetHead{serial.GetRootAsWorkingSet(bs, message.MessagePrefixSz), addr}
}
func (h serialWorkingSetHead) TypeName() string {
@@ -304,13 +305,14 @@ func newHead(head types.Value, addr hash.Hash) (dsHead, error) {
if sm, ok := head.(types.SerialMessage); ok {
data := []byte(sm)
if serial.GetFileID(data) == serial.TagFileID {
fid := serial.GetFileID(data[message.MessagePrefixSz:])
if fid == serial.TagFileID {
return newSerialTagHead(data, addr), nil
}
if serial.GetFileID(data) == serial.WorkingSetFileID {
if fid == serial.WorkingSetFileID {
return newSerialWorkingSetHead(data, addr), nil
}
if serial.GetFileID(data) == serial.CommitFileID {
if fid == serial.CommitFileID {
return newSerialCommitHead(sm, addr), nil
}
}

16
go/store/datas/refmap.go
View File

@@ -15,31 +15,29 @@
package datas
import (
"bytes"
flatbuffers "github.com/google/flatbuffers/go"
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
func storeroot_flatbuffer(am prolly.AddressMap) []byte {
func storeroot_flatbuffer(am prolly.AddressMap) message.Message {
builder := flatbuffers.NewBuilder(1024)
ambytes := []byte(tree.ValueFromNode(am.Node()).(types.TupleRowStorage))
ambytes := []byte(tree.ValueFromNode(am.Node()).(types.SerialMessage))
voff := builder.CreateByteVector(ambytes)
serial.StoreRootStart(builder)
serial.StoreRootAddAddressMap(builder, voff)
builder.FinishWithFileIdentifier(serial.StoreRootEnd(builder), []byte(serial.StoreRootFileID))
return builder.FinishedBytes()
return message.FinishMessage(builder, serial.StoreRootEnd(builder), []byte(serial.StoreRootFileID))
}
func parse_storeroot(bs []byte, ns tree.NodeStore) prolly.AddressMap {
if !bytes.Equal([]byte(serial.StoreRootFileID), bs[4:8]) {
panic("expected store root file id, got: " + string(bs[4:8]))
if serial.GetFileID(bs[message.MessagePrefixSz:]) != serial.StoreRootFileID {
panic("expected store root file id, got: " + serial.GetFileID(bs[message.MessagePrefixSz:]))
}
sr := serial.GetRootAsStoreRoot(bs, 0)
sr := serial.GetRootAsStoreRoot(bs, message.MessagePrefixSz)
mapbytes := sr.AddressMapBytes()
node := tree.NodeFromBytes(mapbytes)
return prolly.NewAddressMap(node, ns)

8
go/store/datas/tag.go
View File

@@ -23,6 +23,7 @@ import (
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/nomdl"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/types"
)
@@ -111,7 +112,7 @@ func newTag(ctx context.Context, db *database, commitAddr hash.Hash, meta *TagMe
}
}
func tag_flatbuffer(commitAddr hash.Hash, meta *TagMeta) []byte {
func tag_flatbuffer(commitAddr hash.Hash, meta *TagMeta) message.Message {
builder := flatbuffers.NewBuilder(1024)
addroff := builder.CreateByteVector(commitAddr[:])
var nameOff, emailOff, descOff flatbuffers.UOffsetT
@@ -129,8 +130,7 @@ func tag_flatbuffer(commitAddr hash.Hash, meta *TagMeta) []byte {
serial.TagAddTimestampMillis(builder, meta.Timestamp)
serial.TagAddUserTimestampMillis(builder, meta.UserTimestamp)
}
builder.FinishWithFileIdentifier(serial.TagEnd(builder), []byte(serial.TagFileID))
return builder.FinishedBytes()
return message.FinishMessage(builder, serial.TagEnd(builder), []byte(serial.TagFileID))
}
func IsTag(v types.Value) (bool, error) {
@@ -138,7 +138,7 @@ func IsTag(v types.Value) (bool, error) {
return types.IsValueSubtypeOf(s.Format(), v, valueTagType)
} else if sm, ok := v.(types.SerialMessage); ok {
data := []byte(sm)
return serial.GetFileID(data) == serial.TagFileID, nil
return serial.GetFileID(data[message.MessagePrefixSz:]) == serial.TagFileID, nil
}
return false, nil
}

9
go/store/datas/workingset.go
View File

@@ -21,6 +21,7 @@ import (
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/types"
)
@@ -148,7 +149,7 @@ func newWorkingSet(ctx context.Context, db *database, meta *WorkingSetMeta, work
return ref.TargetHash(), ref, nil
}
func workingset_flatbuffer(working hash.Hash, staged *hash.Hash, mergeState *MergeState, meta *WorkingSetMeta) []byte {
func workingset_flatbuffer(working hash.Hash, staged *hash.Hash, mergeState *MergeState, meta *WorkingSetMeta) message.Message {
builder := flatbuffers.NewBuilder(1024)
workingoff := builder.CreateByteVector(working[:])
var stagedOff, mergeStateOff flatbuffers.UOffsetT
@@ -185,9 +186,7 @@ func workingset_flatbuffer(working hash.Hash, staged *hash.Hash, mergeState *Mer
serial.WorkingSetAddDesc(builder, descOff)
serial.WorkingSetAddTimestampMillis(builder, meta.Timestamp)
}
builder.FinishWithFileIdentifier(serial.WorkingSetEnd(builder), []byte(serial.WorkingSetFileID))
return builder.FinishedBytes()
return message.FinishMessage(builder, serial.WorkingSetEnd(builder), []byte(serial.WorkingSetFileID))
}
func NewMergeState(ctx context.Context, vrw types.ValueReadWriter, preMergeWorking types.Ref, commit *Commit) (*MergeState, error) {
@@ -222,7 +221,7 @@ func IsWorkingSet(v types.Value) (bool, error) {
// types.IsValueSubtypeOf is very strict about the type description.
return s.Name() == workingSetName, nil
} else if sm, ok := v.(types.SerialMessage); ok {
return serial.GetFileID([]byte(sm)) == serial.WorkingSetFileID, nil
return serial.GetFileID(sm[message.MessagePrefixSz:]) == serial.WorkingSetFileID, nil
} else {
return false, nil
}

2
go/store/prolly/commit_closure_test.go
View File

@@ -148,7 +148,7 @@ func TestCommitClosure(t *testing.T) {
assert.Equal(t, 4096, cc.Count())
// Walk the addresses in the root.
msg := message.Message(tree.ValueFromNode(cc.closure.root).(types.TupleRowStorage))
msg := message.Message(tree.ValueFromNode(cc.closure.root).(types.SerialMessage))
numaddresses := 0
err = message.WalkAddresses(ctx, msg, func(ctx context.Context, addr hash.Hash) error {
numaddresses++

26
go/store/prolly/message/address_map.go
View File

@@ -75,10 +75,10 @@ func (s AddressMapSerializer) Serialize(keys, addrs [][]byte, subtrees []uint64,
}
serial.AddressMapAddTreeLevel(b, uint8(level))
return finishMessage(b, serial.AddressMapEnd(b), addressMapFileID)
return FinishMessage(b, serial.AddressMapEnd(b), addressMapFileID)
}
func finishMessage(b *fb.Builder, off fb.UOffsetT, fileID []byte) []byte {
func FinishMessage(b *fb.Builder, off fb.UOffsetT, fileID []byte) Message {
// We finish the buffer by prefixing it with:
// 1) 1 byte NomsKind == TupleRowStorage.
// 2) big endian uint16 representing the size of the message, not
@@ -88,33 +88,33 @@ func finishMessage(b *fb.Builder, off fb.UOffsetT, fileID []byte) []byte {
// codec.
//
// All accessors in this package expect this prefix to be on the front
// of the message bytes as well. See |messagePrefixSz|.
// of the message bytes as well. See |MessagePrefixSz|.
b.Prep(1, fb.SizeInt32+4+messagePrefixSz)
b.Prep(1, fb.SizeInt32+4+MessagePrefixSz)
b.FinishWithFileIdentifier(off, fileID)
bytes := b.Bytes[b.Head()-messagePrefixSz:]
bytes := b.Bytes[b.Head()-MessagePrefixSz:]
bytes[0] = byte(MessageTypesKind)
binary.BigEndian.PutUint16(bytes[1:], uint16(len(b.Bytes)-int(b.Head())))
return bytes
}
func getAddressMapKeys(msg Message) (keys val.SlicedBuffer) {
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
am := serial.GetRootAsAddressMap(msg, MessagePrefixSz)
keys.Buf = am.KeyItemsBytes()
keys.Offs = getAddressMapKeyOffsets(am)
return
}
func getAddressMapValues(msg Message) (values val.SlicedBuffer) {
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
am := serial.GetRootAsAddressMap(msg, MessagePrefixSz)
values.Buf = am.AddressArrayBytes()
values.Offs = offsetsForAddressArray(values.Buf)
return
}
func walkAddressMapAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error {
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
am := serial.GetRootAsAddressMap(msg, MessagePrefixSz)
arr := am.AddressArrayBytes()
for i := 0; i < len(arr)/hash.ByteLen; i++ {
addr := hash.New(arr[i*addrSize : (i+1)*addrSize])
@@ -126,7 +126,7 @@ func walkAddressMapAddresses(ctx context.Context, msg Message, cb func(ctx conte
}
func getAddressMapCount(msg Message) uint16 {
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
am := serial.GetRootAsAddressMap(msg, MessagePrefixSz)
if am.KeyItemsLength() == 0 {
return 0
}
@@ -135,18 +135,18 @@ func getAddressMapCount(msg Message) uint16 {
}
func getAddressMapTreeLevel(msg Message) int {
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
am := serial.GetRootAsAddressMap(msg, MessagePrefixSz)
return int(am.TreeLevel())
}
func getAddressMapTreeCount(msg Message) int {
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
am := serial.GetRootAsAddressMap(msg, MessagePrefixSz)
return int(am.TreeCount())
}
func getAddressMapSubtrees(msg Message) []uint64 {
counts := make([]uint64, getAddressMapCount(msg))
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
am := serial.GetRootAsAddressMap(msg, MessagePrefixSz)
return decodeVarints(am.SubtreeCountsBytes(), counts)
}
@@ -170,6 +170,6 @@ func estimateAddressMapSize(keys, addresses [][]byte, subtrees []uint64) (keySz,
totalSz += len(subtrees) * binary.MaxVarintLen64
totalSz += 8 + 1 + 1 + 1
totalSz += 72
totalSz += messagePrefixSz
totalSz += MessagePrefixSz
return
}

18
go/store/prolly/message/commit_closure.go
View File

@@ -51,7 +51,7 @@ func offsetsForCommitClosureKeys(buf []byte) []byte {
func getCommitClosureKeys(msg Message) val.SlicedBuffer {
var ret val.SlicedBuffer
m := serial.GetRootAsCommitClosure(msg, messagePrefixSz)
m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz)
ret.Buf = m.KeyItemsBytes()
ret.Offs = offsetsForCommitClosureKeys(ret.Buf)
return ret
@@ -59,7 +59,7 @@ func getCommitClosureKeys(msg Message) val.SlicedBuffer {
func getCommitClosureValues(msg Message) val.SlicedBuffer {
var ret val.SlicedBuffer
m := serial.GetRootAsCommitClosure(msg, messagePrefixSz)
m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz)
if m.AddressArrayLength() == 0 {
ret.Buf = commitClosureEmptyValueBytes
ret.Offs = commitClosureValueOffsets[:getCommitClosureCount(msg)*uint16Size]
@@ -74,28 +74,28 @@ func getCommitClosureValues(msg Message) val.SlicedBuffer {
const commitClosureKeyLength = 8 + 20
func getCommitClosureCount(msg Message) uint16 {
m := serial.GetRootAsCommitClosure(msg, messagePrefixSz)
m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz)
return uint16(m.KeyItemsLength() / commitClosureKeyLength)
}
func getCommitClosureTreeLevel(msg Message) int {
m := serial.GetRootAsCommitClosure(msg, messagePrefixSz)
m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz)
return int(m.TreeLevel())
}
func getCommitClosureTreeCount(msg Message) int {
m := serial.GetRootAsCommitClosure(msg, messagePrefixSz)
m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz)
return int(m.TreeCount())
}
func getCommitClosureSubtrees(msg Message) []uint64 {
counts := make([]uint64, getCommitClosureCount(msg))
m := serial.GetRootAsCommitClosure(msg, messagePrefixSz)
m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz)
return decodeVarints(m.SubtreeCountsBytes(), counts)
}
func walkCommitClosureAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error {
m := serial.GetRootAsCommitClosure(msg, messagePrefixSz)
m := serial.GetRootAsCommitClosure(msg, MessagePrefixSz)
arr := m.AddressArrayBytes()
for i := 0; i < len(arr)/hash.ByteLen; i++ {
addr := hash.New(arr[i*addrSize : (i+1)*addrSize])
@@ -153,7 +153,7 @@ func (s CommitClosureSerializer) Serialize(keys, addrs [][]byte, subtrees []uint
}
serial.CommitClosureAddTreeLevel(b, uint8(level))
return finishMessage(b, serial.CommitClosureEnd(b), commitClosureFileID)
return FinishMessage(b, serial.CommitClosureEnd(b), commitClosureFileID)
}
func estimateCommitClosureSize(keys, addresses [][]byte, subtrees []uint64) (keySz, addrSz, totalSz int) {
@@ -163,6 +163,6 @@ func estimateCommitClosureSize(keys, addresses [][]byte, subtrees []uint64) (key
totalSz += len(subtrees) * binary.MaxVarintLen64
totalSz += 8 + 1 + 1 + 1
totalSz += 72
totalSz += messagePrefixSz
totalSz += MessagePrefixSz
return
}

14
go/store/prolly/message/message.go
View File

@@ -24,9 +24,9 @@ import (
"github.com/dolthub/dolt/go/store/hash"
)
const MessageTypesKind int = 28
const MessageTypesKind int = 27
const messagePrefixSz = 3
const MessagePrefixSz = 3
type Message []byte
@@ -35,7 +35,7 @@ type Serializer interface {
}
func GetKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt uint16) {
id := serial.GetFileID(msg[messagePrefixSz:])
id := serial.GetFileID(msg[MessagePrefixSz:])
if id == serial.ProllyTreeNodeFileID {
return getProllyMapKeysAndValues(msg)
@@ -57,7 +57,7 @@ func GetKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt uint16) {
}
func WalkAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error {
id := serial.GetFileID(msg[messagePrefixSz:])
id := serial.GetFileID(msg[MessagePrefixSz:])
switch id {
case serial.ProllyTreeNodeFileID:
return walkProllyMapAddresses(ctx, msg, cb)
@@ -71,7 +71,7 @@ func WalkAddresses(ctx context.Context, msg Message, cb func(ctx context.Context
}
func GetTreeLevel(msg Message) int {
id := serial.GetFileID(msg[messagePrefixSz:])
id := serial.GetFileID(msg[MessagePrefixSz:])
switch id {
case serial.ProllyTreeNodeFileID:
return getProllyMapTreeLevel(msg)
@@ -85,7 +85,7 @@ func GetTreeLevel(msg Message) int {
}
func GetTreeCount(msg Message) int {
id := serial.GetFileID(msg[messagePrefixSz:])
id := serial.GetFileID(msg[MessagePrefixSz:])
switch id {
case serial.ProllyTreeNodeFileID:
return getProllyMapTreeCount(msg)
@@ -99,7 +99,7 @@ func GetTreeCount(msg Message) int {
}
func GetSubtrees(msg Message) []uint64 {
id := serial.GetFileID(msg[messagePrefixSz:])
id := serial.GetFileID(msg[MessagePrefixSz:])
switch id {
case serial.ProllyTreeNodeFileID:
return getProllyMapSubtrees(msg)

16
go/store/prolly/message/prolly_map.go
View File

@@ -93,11 +93,11 @@ func (s ProllyMapSerializer) Serialize(keys, values [][]byte, subtrees []uint64,
serial.ProllyTreeNodeAddValueType(b, serial.ItemTypeTupleFormatAlpha)
serial.ProllyTreeNodeAddTreeLevel(b, uint8(level))
return finishMessage(b, serial.ProllyTreeNodeEnd(b), prollyMapFileID)
return FinishMessage(b, serial.ProllyTreeNodeEnd(b), prollyMapFileID)
}
func getProllyMapKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt uint16) {
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz)
keys.Buf = pm.KeyItemsBytes()
keys.Offs = getProllyMapKeyOffsets(pm)
@@ -120,7 +120,7 @@ func getProllyMapKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt
}
func walkProllyMapAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error {
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz)
arr := pm.AddressArrayBytes()
for i := 0; i < len(arr)/hash.ByteLen; i++ {
addr := hash.New(arr[i*addrSize : (i+1)*addrSize])
@@ -143,7 +143,7 @@ func walkProllyMapAddresses(ctx context.Context, msg Message, cb func(ctx contex
}
func getProllyMapCount(msg Message) uint16 {
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz)
if pm.KeyItemsLength() == 0 {
return 0
}
@@ -152,18 +152,18 @@ func getProllyMapCount(msg Message) uint16 {
}
func getProllyMapTreeLevel(msg Message) int {
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz)
return int(pm.TreeLevel())
}
func getProllyMapTreeCount(msg Message) int {
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz)
return int(pm.TreeCount())
}
func getProllyMapSubtrees(msg Message) []uint64 {
counts := make([]uint64, getProllyMapCount(msg))
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
pm := serial.GetRootAsProllyTreeNode(msg, MessagePrefixSz)
return decodeVarints(pm.SubtreeCountsBytes(), counts)
}
@@ -213,7 +213,7 @@ func estimateProllyMapSize(keys, values [][]byte, subtrees []uint64, valAddrsCnt
bufSz += 72 // vtable (approx)
bufSz += 100 // padding?
bufSz += valAddrsCnt * len(values)
bufSz += messagePrefixSz
bufSz += MessagePrefixSz
return keySz, valSz, bufSz
}

2
go/store/prolly/shim/shim.go
View File

@@ -25,7 +25,7 @@ import (
)
func NodeFromValue(v types.Value) tree.Node {
return tree.NodeFromBytes(v.(types.TupleRowStorage))
return tree.NodeFromBytes(v.(types.SerialMessage))
}
func ValueFromMap(m prolly.Map) types.Value {

9
go/store/prolly/tree/content_address_test.go
View File

@@ -27,11 +27,10 @@ import (
)
var goldenHash = hash.Hash{
0x39, 0x1c, 0xcb, 0xd8,
0xea, 0xd2, 0xdc, 0x42,
0x76, 0xb7, 0x38, 0xf1,
0x0d, 0x4f, 0x48, 0x91,
0x7d, 0x1f, 0xb7, 0xb4,
0xde, 0x2c, 0x34, 0x6a, 0xb3,
0x88, 0x6f, 0x95, 0xb, 0x3b,
0x77, 0xa1, 0x1f, 0xfa, 0x4c,
0xf1, 0xe8, 0xdb, 0xfa, 0xcc,
}
// todo(andy): need and analogous test in pkg prolly

2
go/store/prolly/tree/node.go
View File

@@ -235,5 +235,5 @@ func OutputAddressMapNode(w io.Writer, node Node) error {
}
func ValueFromNode(root Node) types.Value {
return types.TupleRowStorage(root.bytes())
return types.SerialMessage(root.bytes())
}

2
go/store/prolly/tree/node_test.go
View File

@@ -104,7 +104,7 @@ func TestNodeDecodeValueCompatibility(t *testing.T) {
v, err := vs.ReadValue(context.Background(), h)
require.NoError(t, err)
assert.Equal(t, nd.bytes(), []byte(v.(types.TupleRowStorage)))
assert.Equal(t, nd.bytes(), []byte(v.(types.SerialMessage)))
}
func randomNodeItemPairs(t *testing.T, count int) (keys, values []Item) {

2
go/store/types/codec.go
View File

@@ -325,7 +325,7 @@ func (b *binaryNomsReader) ReadInlineBlob() []byte {
return bytes
}
func (b *binaryNomsReader) readTupleRowStorage() []byte {
func (b *binaryNomsReader) readSerialMessage() []byte {
size := uint32(b.readUint16())
// start at offset-3, to include the kind byte + Uint16 for size...
bytes := b.buff[b.offset-3 : b.offset+size]

64
go/store/types/noms_kind.go
View File

@@ -66,7 +66,6 @@ const (
PolygonKind
SerialMessageKind
TupleRowStorageKind
UnknownKind NomsKind = 255
)
@@ -99,7 +98,6 @@ func init() {
KindToType[LineStringKind] = LineString{}
KindToType[PolygonKind] = Polygon{}
KindToType[SerialMessageKind] = SerialMessage{}
KindToType[TupleRowStorageKind] = TupleRowStorage{}
SupportedKinds[BlobKind] = true
SupportedKinds[BoolKind] = true
@@ -128,45 +126,43 @@ func init() {
SupportedKinds[LineStringKind] = true
SupportedKinds[PolygonKind] = true
SupportedKinds[SerialMessageKind] = true
SupportedKinds[TupleRowStorageKind] = true
if message.MessageTypesKind != int(TupleRowStorageKind) {
panic("internal error: message.MessageTypesKind != TupleRowStorageKind")
if message.MessageTypesKind != int(SerialMessageKind) {
panic("internal error: message.MessageTypesKind != SerialMessageKind")
}
}
var KindToTypeSlice []Value
var KindToString = map[NomsKind]string{
UnknownKind: "unknown",
BlobKind: "Blob",
BoolKind: "Bool",
CycleKind: "Cycle",
ListKind: "List",
MapKind: "Map",
FloatKind: "Float",
RefKind: "Ref",
SetKind: "Set",
StructKind: "Struct",
StringKind: "String",
TypeKind: "Type",
UnionKind: "Union",
ValueKind: "Value",
UUIDKind: "UUID",
IntKind: "Int",
UintKind: "Uint",
NullKind: "Null",
TupleKind: "Tuple",
InlineBlobKind: "InlineBlob",
TimestampKind: "Timestamp",
DecimalKind: "Decimal",
JSONKind: "JSON",
GeometryKind: "Geometry",
PointKind: "Point",
LineStringKind: "LineString",
PolygonKind: "Polygon",
SerialMessageKind: "SerialMessage",
TupleRowStorageKind: "TupleRowStorage",
UnknownKind: "unknown",
BlobKind: "Blob",
BoolKind: "Bool",
CycleKind: "Cycle",
ListKind: "List",
MapKind: "Map",
FloatKind: "Float",
RefKind: "Ref",
SetKind: "Set",
StructKind: "Struct",
StringKind: "String",
TypeKind: "Type",
UnionKind: "Union",
ValueKind: "Value",
UUIDKind: "UUID",
IntKind: "Int",
UintKind: "Uint",
NullKind: "Null",
TupleKind: "Tuple",
InlineBlobKind: "InlineBlob",
TimestampKind: "Timestamp",
DecimalKind: "Decimal",
JSONKind: "JSON",
GeometryKind: "Geometry",
PointKind: "Point",
LineStringKind: "LineString",
PolygonKind: "Polygon",
SerialMessageKind: "SerialMessage",
}
// String returns the name of the kind.

6
go/store/types/path.go
View File

@@ -34,6 +34,7 @@ import (
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/d"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/message"
)
// For an annotation like @type, 1st capture group is the annotation.
@@ -235,9 +236,8 @@ func (fp FieldPath) Resolve(ctx context.Context, v Value, vr ValueReader) (Value
return sv, nil
}
case SerialMessage:
data := []byte(v)
if serial.GetFileID(data) == serial.CommitFileID && fp.Name == "value" {
msg := serial.GetRootAsCommit(data, 0)
if serial.GetFileID(v[message.MessagePrefixSz:]) == serial.CommitFileID && fp.Name == "value" {
msg := serial.GetRootAsCommit(v, message.MessagePrefixSz)
addr := hash.New(msg.RootBytes())
return vr.ReadValue(ctx, addr)
}

91
go/store/types/serial_message.go
View File

@@ -17,13 +17,14 @@ package types
import (
"bytes"
"context"
"encoding/hex"
"fmt"
"math"
"strings"
"time"
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/message"
)
type SerialMessage []byte
@@ -52,17 +53,18 @@ func (sm SerialMessage) Hash(nbf *NomsBinFormat) (hash.Hash, error) {
}
func (sm SerialMessage) HumanReadableString() string {
switch serial.GetFileID(sm) {
id := serial.GetFileID(sm[message.MessagePrefixSz:])
switch id {
case serial.StoreRootFileID:
msg := serial.GetRootAsStoreRoot([]byte(sm), 0)
msg := serial.GetRootAsStoreRoot([]byte(sm), message.MessagePrefixSz)
ret := &strings.Builder{}
mapbytes := msg.AddressMapBytes()
fmt.Fprintf(ret, "StoreRoot{%s}", TupleRowStorage(mapbytes).HumanReadableString())
fmt.Fprintf(ret, "StoreRoot{%s}", SerialMessage(mapbytes).HumanReadableString())
return ret.String()
case serial.TagFileID:
return "Tag"
case serial.WorkingSetFileID:
msg := serial.GetRootAsWorkingSet(sm, 0)
msg := serial.GetRootAsWorkingSet(sm, message.MessagePrefixSz)
ret := &strings.Builder{}
fmt.Fprintf(ret, "{\n")
fmt.Fprintf(ret, "\tName: %s\n", msg.Name())
@@ -74,7 +76,7 @@ func (sm SerialMessage) HumanReadableString() string {
fmt.Fprintf(ret, "}")
return ret.String()
case serial.CommitFileID:
msg := serial.GetRootAsCommit(sm, 0)
msg := serial.GetRootAsCommit(sm, message.MessagePrefixSz)
ret := &strings.Builder{}
fmt.Fprintf(ret, "{\n")
fmt.Fprintf(ret, "\tName: %s\n", msg.Name())
@@ -102,18 +104,18 @@ func (sm SerialMessage) HumanReadableString() string {
fmt.Fprintf(ret, "}")
return ret.String()
case serial.RootValueFileID:
msg := serial.GetRootAsRootValue(sm, 0)
msg := serial.GetRootAsRootValue(sm, message.MessagePrefixSz)
ret := &strings.Builder{}
fmt.Fprintf(ret, "{\n")
fmt.Fprintf(ret, "\tFeatureVersion: %d\n", msg.FeatureVersion())
fmt.Fprintf(ret, "\tForeignKeys: #%s\n", hash.New(msg.ForeignKeyAddrBytes()).String())
fmt.Fprintf(ret, "\tSuperSchema: #%s\n", hash.New(msg.SuperSchemasAddrBytes()).String())
fmt.Fprintf(ret, "\tTables: {\n\t%s", TupleRowStorage(msg.TablesBytes()).HumanReadableString())
fmt.Fprintf(ret, "\tTables: {\n\t%s", SerialMessage(msg.TablesBytes()).HumanReadableString())
fmt.Fprintf(ret, "\t}\n")
fmt.Fprintf(ret, "}")
return ret.String()
case serial.TableFileID:
msg := serial.GetRootAsTable(sm, 0)
msg := serial.GetRootAsTable(sm, message.MessagePrefixSz)
ret := &strings.Builder{}
fmt.Fprintf(ret, "{\n")
@@ -126,16 +128,27 @@ func (sm SerialMessage) HumanReadableString() string {
// TODO: can't use tree package to print here, creates a cycle
fmt.Fprintf(ret, "\tPrimary index: prolly tree\n")
fmt.Fprintf(ret, "\tSecondary indexes: {\n\t%s\n", TupleRowStorage(msg.SecondaryIndexesBytes()).HumanReadableString())
fmt.Fprintf(ret, "\tSecondary indexes: {\n\t%s\n", SerialMessage(msg.SecondaryIndexesBytes()).HumanReadableString())
fmt.Fprintf(ret, "\t}\n")
fmt.Fprintf(ret, "}")
return ret.String()
case serial.ProllyTreeNodeFileID:
return "ProllyTreeNode"
case serial.AddressMapFileID:
return "AddressMap"
keys, values, cnt := message.GetKeysAndValues(message.Message(sm))
var b strings.Builder
b.Write([]byte("AddressMap{\n"))
for i := uint16(0); i < cnt; i++ {
name := keys.GetSlice(int(i))
addr := values.GetSlice(int(i))
b.Write([]byte("\t"))
b.Write(name)
b.Write([]byte(": "))
b.Write([]byte(hash.New(addr).String()))
b.Write([]byte("\n"))
}
b.Write([]byte("}"))
return b.String()
default:
return "SerialMessage (HumanReadableString not implemented)"
return fmt.Sprintf("SerialMessage (HumanReadableString not implemented), [%v]: %s", id, strings.ToUpper(hex.EncodeToString(sm)))
}
}
@@ -151,15 +164,15 @@ func (sm SerialMessage) Less(nbf *NomsBinFormat, other LesserValuable) (bool, er
const SerialMessageRefHeight = 1024
func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
switch serial.GetFileID([]byte(sm)) {
switch serial.GetFileID(sm[message.MessagePrefixSz:]) {
case serial.StoreRootFileID:
msg := serial.GetRootAsStoreRoot([]byte(sm), 0)
msg := serial.GetRootAsStoreRoot([]byte(sm), message.MessagePrefixSz)
if msg.AddressMapLength() > 0 {
mapbytes := msg.AddressMapBytes()
return TupleRowStorage(mapbytes).walkRefs(nbf, cb)
return SerialMessage(mapbytes).walkRefs(nbf, cb)
}
case serial.TagFileID:
msg := serial.GetRootAsTag([]byte(sm), 0)
msg := serial.GetRootAsTag([]byte(sm), message.MessagePrefixSz)
addr := hash.New(msg.CommitAddrBytes())
r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight)
if err != nil {
@@ -167,7 +180,7 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
}
return cb(r)
case serial.WorkingSetFileID:
msg := serial.GetRootAsWorkingSet([]byte(sm), 0)
msg := serial.GetRootAsWorkingSet([]byte(sm), message.MessagePrefixSz)
addr := hash.New(msg.WorkingRootAddrBytes())
r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight)
if err != nil {
@@ -207,8 +220,8 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
}
}
case serial.RootValueFileID:
msg := serial.GetRootAsRootValue([]byte(sm), 0)
err := TupleRowStorage(msg.TablesBytes()).walkRefs(nbf, cb)
msg := serial.GetRootAsRootValue([]byte(sm), message.MessagePrefixSz)
err := SerialMessage(msg.TablesBytes()).walkRefs(nbf, cb)
if err != nil {
return err
}
@@ -233,7 +246,7 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
}
}
case serial.TableFileID:
msg := serial.GetRootAsTable([]byte(sm), 0)
msg := serial.GetRootAsTable([]byte(sm), message.MessagePrefixSz)
addr := hash.New(msg.SchemaBytes())
r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight)
if err != nil {
@@ -300,7 +313,7 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
}
}
err = TupleRowStorage(msg.SecondaryIndexesBytes()).walkRefs(nbf, cb)
err = SerialMessage(msg.SecondaryIndexesBytes()).walkRefs(nbf, cb)
if err != nil {
return err
}
@@ -315,7 +328,7 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
}
return v.walkRefs(nbf, cb)
} else {
return TupleRowStorage(mapbytes).walkRefs(nbf, cb)
return SerialMessage(mapbytes).walkRefs(nbf, cb)
}
case serial.CommitFileID:
parents, err := SerialCommitParentAddrs(nbf, sm)
@@ -331,7 +344,7 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
return err
}
}
msg := serial.GetRootAsCommit([]byte(sm), 0)
msg := serial.GetRootAsCommit([]byte(sm), message.MessagePrefixSz)
addr := hash.New(msg.RootBytes())
r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight)
if err != nil {
@@ -355,14 +368,26 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
return nil
case serial.ForeignKeyCollectionFileID:
return nil
case serial.ProllyTreeNodeFileID:
fallthrough
case serial.AddressMapFileID:
fallthrough
case serial.CommitClosureFileID:
return message.WalkAddresses(context.TODO(), message.Message(sm), func(ctx context.Context, addr hash.Hash) error {
r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight)
if err != nil {
return err
}
return cb(r)
})
default:
return fmt.Errorf("unsupported SerialMessage message with FileID: %s", serial.GetFileID([]byte(sm)))
return fmt.Errorf("unsupported SerialMessage message with FileID: %s", serial.GetFileID(sm[message.MessagePrefixSz:]))
}
return nil
}
func SerialCommitParentAddrs(nbf *NomsBinFormat, sm SerialMessage) ([]hash.Hash, error) {
msg := serial.GetRootAsCommit([]byte(sm), 0)
msg := serial.GetRootAsCommit([]byte(sm), message.MessagePrefixSz)
addrs := msg.ParentAddrsBytes()
n := len(addrs) / 20
ret := make([]hash.Hash, n)
@@ -375,7 +400,7 @@ func SerialCommitParentAddrs(nbf *NomsBinFormat, sm SerialMessage) ([]hash.Hash,
}
func (sm SerialMessage) readFrom(nbf *NomsBinFormat, b *binaryNomsReader) (Value, error) {
bytes := b.ReadInlineBlob()
bytes := b.readSerialMessage()
return SerialMessage(bytes), nil
}
@@ -389,16 +414,6 @@ func (sm SerialMessage) typeOf() (*Type, error) {
}
func (sm SerialMessage) writeTo(w nomsWriter, nbf *NomsBinFormat) error {
byteLen := len(sm)
if byteLen > math.MaxUint16 {
return fmt.Errorf("SerialMessage has length %v when max is %v", byteLen, math.MaxUint16)
}
err := SerialMessageKind.writeTo(w, nbf)
if err != nil {
return err
}
w.writeUint16(uint16(byteLen))
w.writeRaw(sm)
return nil
}

122
go/store/types/tuplerowstorage.go
View File

@@ -1,122 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"bytes"
"context"
"encoding/hex"
"strings"
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/message"
)
// TupleRowStorage cribs its implementation from InlineBlob. It bridges
// prolly/message byte arrays between types.Value and prolly/tree.Node.
//
// Unlike SerialMessage, the byte array held in TupleRowStorage includes the
// NomsKind byte and the BigEndian uint16 size of the message. |writeTo| is
// simply a call through to writeRaw, and |readFrom| has to pick up bytes from
// the reader that have already been "read" to determine kind and size.
type TupleRowStorage []byte
func (v TupleRowStorage) Value(ctx context.Context) (Value, error) {
return v, nil
}
func (v TupleRowStorage) Equals(other Value) bool {
v2, ok := other.(TupleRowStorage)
if !ok {
return false
}
return bytes.Equal(v, v2)
}
func (v TupleRowStorage) Less(nbf *NomsBinFormat, other LesserValuable) (bool, error) {
if v2, ok := other.(TupleRowStorage); ok {
return bytes.Compare(v, v2) == -1, nil
}
return TupleRowStorageKind < other.Kind(), nil
}
func (v TupleRowStorage) Hash(nbf *NomsBinFormat) (hash.Hash, error) {
return getHash(v, nbf)
}
func (v TupleRowStorage) isPrimitive() bool {
return true
}
func (v TupleRowStorage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
return message.WalkAddresses(context.TODO(), message.Message([]byte(v)), func(ctx context.Context, addr hash.Hash) error {
r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight)
if err != nil {
return err
}
return cb(r)
})
}
func (v TupleRowStorage) typeOf() (*Type, error) {
return PrimitiveTypeMap[TupleRowStorageKind], nil
}
func (v TupleRowStorage) Kind() NomsKind {
return TupleRowStorageKind
}
func (v TupleRowStorage) valueReadWriter() ValueReadWriter {
return nil
}
func (v TupleRowStorage) writeTo(w nomsWriter, nbf *NomsBinFormat) error {
w.writeRaw(v)
return nil
}
func (v TupleRowStorage) readFrom(nbf *NomsBinFormat, b *binaryNomsReader) (Value, error) {
bytes := b.readTupleRowStorage()
return TupleRowStorage(bytes), nil
}
func (v TupleRowStorage) skip(nbf *NomsBinFormat, b *binaryNomsReader) {
size := uint32(b.readUint16())
b.skipBytes(size)
}
func (v TupleRowStorage) HumanReadableString() string {
if serial.GetFileID(v) == serial.AddressMapFileID {
keys, values, cnt := message.GetKeysAndValues(message.Message([]byte(v)))
var b strings.Builder
b.Write([]byte("AddressMap{\n"))
for i := uint16(0); i < cnt; i++ {
name := keys.GetSlice(int(i))
addr := values.GetSlice(int(i))
b.Write([]byte("\t"))
b.Write(name)
b.Write([]byte(": "))
b.Write([]byte(hash.New(addr).String()))
b.Write([]byte("\n"))
}
b.Write([]byte("}"))
return b.String()
} else {
return strings.ToUpper(hex.EncodeToString(v))
}
}