mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-02-15 00:50:22 -06:00
Code Issues Packages Projects Releases 132 Wiki Activity

added file identifier for prolly nodes

Browse Source
This commit is contained in:
Andy Arthur
2022-04-29 15:42:59 -07:00
parent c9bee38f9f
commit 63e666a2ba
12 changed files with 30 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
go/gen/fb/serial/fileidentifiers.go
View File

@@ -22,6 +22,7 @@ const WorkingSetFileID = "WRST"
const CommitFileID = "DCMT"
const RootValueFileID = "RTVL"
const TableFileID = "DTBL"
const ProllyTreeNodeFileID = "PRLY"
func GetFileID(bs []byte) string {
if len(bs) < 8 {

1
go/serial/fileidentifiers.go
View File

@@ -22,6 +22,7 @@ const WorkingSetFileID = "WRST"
const CommitFileID = "DCMT"
const RootValueFileID = "RTVL"
const TableFileID = "DTBL"
const ProllyTreeNodeFileID = "PRLY"
func GetFileID(bs []byte) string {
if len(bs) < 8 {

6
go/serial/prolly.fbs
View File

@@ -81,3 +81,9 @@ table CommitClosure {
// node tree level, 0 for leaf nodes
tree_level:uint8;
}
// KEEP THIS IN SYNC WITH fileidentifiers.go
file_identifier "PRLY";
root_type ProllyTreeNode;

2
go/store/prolly/address_map.go
View File

@@ -235,7 +235,7 @@ func (nb *addrMapBuilder) Build(pool pool.BuffPool) (node tree.Node) {
serial.ProllyTreeNodeAddTreeCount(b, uint64(len(nb.keys)))
}
serial.ProllyTreeNodeAddTreeLevel(b, uint8(nb.level))
b.Finish(serial.ProllyTreeNodeEnd(b))
b.FinishWithFileIdentifier(serial.ProllyTreeNodeEnd(b), []byte(serial.ProllyTreeNodeFileID))
nb.reset()
buf := b.FinishedBytes()

2
go/store/prolly/map.go
View File

@@ -335,7 +335,7 @@ func (nb *mapBuilder) Build(pool pool.BuffPool) (node tree.Node) {
serial.ProllyTreeNodeAddKeyType(b, serial.ItemTypeTupleFormatAlpha)
serial.ProllyTreeNodeAddValueType(b, serial.ItemTypeTupleFormatAlpha)
serial.ProllyTreeNodeAddTreeLevel(b, uint8(nb.level))
b.Finish(serial.ProllyTreeNodeEnd(b))
b.FinishWithFileIdentifier(serial.ProllyTreeNodeEnd(b), []byte(serial.ProllyTreeNodeFileID))
nb.reset()
buf := b.FinishedBytes()

2
go/store/prolly/map_test.go
View File

@@ -93,8 +93,8 @@ func TestNewEmptyNode(t *testing.T) {
empty := newEmptyMapNode(sharedPool)
assert.Equal(t, 0, empty.Level())
assert.Equal(t, 0, empty.Count())
assert.Equal(t, 68, empty.Size())
assert.Equal(t, 0, empty.TreeCount())
assert.Equal(t, 72, empty.Size())
assert.True(t, empty.IsLeaf())
}

4
go/store/prolly/tree/chunker.go
View File

@@ -457,7 +457,7 @@ func getCanonicalRoot(ctx context.Context, ns NodeStore, builder NodeBuilder) (N
assertTrue(cnt == 1)
nd := builder.Build(ns.Pool())
mt := nd.getChildAddress(0)
mt := nd.getAddress(0)
for {
child, err := fetchChild(ctx, ns, mt)
@@ -469,6 +469,6 @@ func getCanonicalRoot(ctx context.Context, ns NodeStore, builder NodeBuilder) (N
return child, nil
}
mt = child.getChildAddress(0)
mt = child.getAddress(0)
}
}

8
go/store/prolly/tree/content_address_test.go
View File

@@ -26,10 +26,10 @@ import (
)
var goldenHash = hash.Hash{
0x9c, 0xf1, 0x55, 0xce, 0x1c,
0xfc, 0x6e, 0xd5, 0x4b, 0xa9,
0xcc, 0x7e, 0x15, 0x93, 0xa8,
0x29, 0x47, 0x16, 0xa8, 0x89,
0xab, 0x85, 0xcf, 0x59, 0x49,
0x5d, 0x32, 0x8, 0xbf, 0x63,
0xad, 0xc3, 0x78, 0x80, 0xc6,
0xa2, 0xd4, 0x7e, 0x40, 0xae,
}
// todo(andy): need and analogous test in pkg prolly

27
go/store/prolly/tree/node.go
View File

@@ -133,7 +133,7 @@ func (nd Node) Level() int {
// IsLeaf returns whether this node is a leaf
func (nd Node) IsLeaf() bool {
return int(nd.msg.TreeLevel()) == 0
return nd.Level() == 0
}
// GetKey returns the |ith| key of this node
@@ -141,30 +141,25 @@ func (nd Node) GetKey(i int) Item {
return nd.keys.GetSlice(i)
}
// getValue returns the |ith| value of this node. Only Valid for leaf nodes.
// getValue returns the |ith| value of this node.
func (nd Node) getValue(i int) Item {
// todo(andy): abstract value access
if nd.values.Buf != nil {
return nd.values.GetSlice(i)
} else {
r := nd.getChildAddress(i)
r := nd.getAddress(i)
return r[:]
}
}
// getChildAddress returns the |ith| address in this node
func (nd Node) getChildAddress(i int) hash.Hash {
// getAddress returns the |ith| address of this node.
// This method assumes values are 20-byte address hashes.
func (nd Node) getAddress(i int) hash.Hash {
refs := nd.msg.AddressArrayBytes()
start, stop := i*addrSz, (i+1)*addrSz
return hash.New(refs[start:stop])
}
// getValueAddress returns the |ith| value address in this node
func (nd Node) getValueAddress(i int) hash.Hash {
o := nd.msg.ValueAddressOffsets(i)
return hash.New(nd.values.Buf[o : o+addrSz])
}
func (nd Node) getSubtreeCounts() SubtreeCounts {
arr := nd.msg.SubtreeCountsBytes()
return readSubtreeCounts(int(nd.count), arr)
@@ -182,19 +177,19 @@ func walkAddresses(ctx context.Context, nd Node, cb AddressCb) (err error) {
arr := nd.msg.AddressArrayBytes()
cnt := len(arr) / addrSz
for i := 0; i < cnt; i++ {
if err = cb(ctx, nd.getChildAddress(i)); err != nil {
if err = cb(ctx, nd.getAddress(i)); err != nil {
return err
}
}
cnt2 := nd.msg.ValueAddressOffsetsLength()
for i := 0; i < cnt2; i++ {
if err = cb(ctx, nd.getValueAddress(i)); err != nil {
o := nd.msg.ValueAddressOffsets(i)
addr := hash.New(nd.values.Buf[o : o+addrSz])
if err = cb(ctx, addr); err != nil {
return err
}
}
assertFalse((cnt > 0) && (cnt2 > 0))
return
}
@@ -250,7 +245,7 @@ func OutputProllyNode(w io.Writer, node Node) error {
w.Write([]byte(" }"))
} else {
ref := node.getChildAddress(i)
ref := hash.New(node.getValue(i))
w.Write([]byte(" ref: #"))
w.Write([]byte(ref.String()))

8
go/store/prolly/tree/node_cursor.go
View File

@@ -219,7 +219,7 @@ func (cur *Cursor) CurrentValue() Item {
}
func (cur *Cursor) CurrentRef() hash.Hash {
return cur.nd.getChildAddress(cur.idx)
return cur.nd.getAddress(cur.idx)
}
func (cur *Cursor) currentSubtreeSize() uint64 {
@@ -489,9 +489,3 @@ func assertTrue(b bool) {
panic("assertion failed")
}
}
func assertFalse(b bool) {
if b {
panic("assertion failed")
}
}

20
go/store/prolly/tree/node_test.go
View File

@@ -62,26 +62,6 @@ func TestRoundTripNodeItems(t *testing.T) {
}
}
func TestGetKeyValueOffsetsVectors(t *testing.T) {
for trial := 0; trial < 100; trial++ {
keys, values := randomNodeItemPairs(t, (rand.Int()%101)+50)
require.True(t, sumSize(keys)+sumSize(values) < MaxVectorOffset)
nd := newLeafNode(keys, values)
ko1, vo1 := offsetsFromSlicedBuffers(nd.keys, nd.values)
ko2, vo2 := offsetsFromFlatbuffer(nd.msg)
assert.Equal(t, len(ko1), len(ko2))
assert.Equal(t, len(ko1), len(keys)-1)
assert.Equal(t, ko1, ko2)
assert.Equal(t, len(vo1), len(vo2))
assert.Equal(t, len(vo1), len(values)-1)
assert.Equal(t, vo1, vo2)
}
}
func TestNodeSize(t *testing.T) {
sz := unsafe.Sizeof(Node{})
assert.Equal(t, 136, int(sz))

2
go/store/prolly/tree/testutils.go
View File

@@ -307,7 +307,7 @@ func (tb *testBuilder) Build(pool pool.BuffPool) (node Node) {
serial.ProllyTreeNodeAddKeyType(b, serial.ItemTypeTupleFormatAlpha)
serial.ProllyTreeNodeAddValueType(b, serial.ItemTypeTupleFormatAlpha)
serial.ProllyTreeNodeAddTreeLevel(b, uint8(tb.level))
b.Finish(serial.ProllyTreeNodeEnd(b))
b.FinishWithFileIdentifier(serial.ProllyTreeNodeEnd(b), []byte(serial.ProllyTreeNodeFileID))
tb.reset()
buf := b.FinishedBytes()