From 63e666a2bae5494a9d1266544015cf37ef6faef1 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 29 Apr 2022 15:42:59 -0700 Subject: [PATCH] added file identifier for prolly nodes --- go/gen/fb/serial/fileidentifiers.go | 1 + go/serial/fileidentifiers.go | 1 + go/serial/prolly.fbs | 6 +++++ go/store/prolly/address_map.go | 2 +- go/store/prolly/map.go | 2 +- go/store/prolly/map_test.go | 2 +- go/store/prolly/tree/chunker.go | 4 +-- go/store/prolly/tree/content_address_test.go | 8 +++--- go/store/prolly/tree/node.go | 27 ++++++++------------ go/store/prolly/tree/node_cursor.go | 8 +----- go/store/prolly/tree/node_test.go | 20 --------------- go/store/prolly/tree/testutils.go | 2 +- 12 files changed, 30 insertions(+), 53 deletions(-) diff --git a/go/gen/fb/serial/fileidentifiers.go b/go/gen/fb/serial/fileidentifiers.go index 50a10f6383..829906813b 100644 --- a/go/gen/fb/serial/fileidentifiers.go +++ b/go/gen/fb/serial/fileidentifiers.go @@ -22,6 +22,7 @@ const WorkingSetFileID = "WRST" const CommitFileID = "DCMT" const RootValueFileID = "RTVL" const TableFileID = "DTBL" +const ProllyTreeNodeFileID = "PRLY" func GetFileID(bs []byte) string { if len(bs) < 8 { diff --git a/go/serial/fileidentifiers.go b/go/serial/fileidentifiers.go index 50a10f6383..829906813b 100644 --- a/go/serial/fileidentifiers.go +++ b/go/serial/fileidentifiers.go @@ -22,6 +22,7 @@ const WorkingSetFileID = "WRST" const CommitFileID = "DCMT" const RootValueFileID = "RTVL" const TableFileID = "DTBL" +const ProllyTreeNodeFileID = "PRLY" func GetFileID(bs []byte) string { if len(bs) < 8 { diff --git a/go/serial/prolly.fbs b/go/serial/prolly.fbs index 7ef323fe09..34b9e0409f 100644 --- a/go/serial/prolly.fbs +++ b/go/serial/prolly.fbs @@ -81,3 +81,9 @@ table CommitClosure { // node tree level, 0 for leaf nodes tree_level:uint8; } + + +// KEEP THIS IN SYNC WITH fileidentifiers.go +file_identifier "PRLY"; + +root_type ProllyTreeNode; diff --git a/go/store/prolly/address_map.go b/go/store/prolly/address_map.go index 046aa382dc..22e32ac194 100644 --- a/go/store/prolly/address_map.go +++ b/go/store/prolly/address_map.go @@ -235,7 +235,7 @@ func (nb *addrMapBuilder) Build(pool pool.BuffPool) (node tree.Node) { serial.ProllyTreeNodeAddTreeCount(b, uint64(len(nb.keys))) } serial.ProllyTreeNodeAddTreeLevel(b, uint8(nb.level)) - b.Finish(serial.ProllyTreeNodeEnd(b)) + b.FinishWithFileIdentifier(serial.ProllyTreeNodeEnd(b), []byte(serial.ProllyTreeNodeFileID)) nb.reset() buf := b.FinishedBytes() diff --git a/go/store/prolly/map.go b/go/store/prolly/map.go index 29dcb515b4..c2b5b1a613 100644 --- a/go/store/prolly/map.go +++ b/go/store/prolly/map.go @@ -335,7 +335,7 @@ func (nb *mapBuilder) Build(pool pool.BuffPool) (node tree.Node) { serial.ProllyTreeNodeAddKeyType(b, serial.ItemTypeTupleFormatAlpha) serial.ProllyTreeNodeAddValueType(b, serial.ItemTypeTupleFormatAlpha) serial.ProllyTreeNodeAddTreeLevel(b, uint8(nb.level)) - b.Finish(serial.ProllyTreeNodeEnd(b)) + b.FinishWithFileIdentifier(serial.ProllyTreeNodeEnd(b), []byte(serial.ProllyTreeNodeFileID)) nb.reset() buf := b.FinishedBytes() diff --git a/go/store/prolly/map_test.go b/go/store/prolly/map_test.go index f6b3e7afae..6def5ab9fb 100644 --- a/go/store/prolly/map_test.go +++ b/go/store/prolly/map_test.go @@ -93,8 +93,8 @@ func TestNewEmptyNode(t *testing.T) { empty := newEmptyMapNode(sharedPool) assert.Equal(t, 0, empty.Level()) assert.Equal(t, 0, empty.Count()) - assert.Equal(t, 68, empty.Size()) assert.Equal(t, 0, empty.TreeCount()) + assert.Equal(t, 72, empty.Size()) assert.True(t, empty.IsLeaf()) } diff --git a/go/store/prolly/tree/chunker.go b/go/store/prolly/tree/chunker.go index 04e86e17f0..fec8200a8b 100644 --- a/go/store/prolly/tree/chunker.go +++ b/go/store/prolly/tree/chunker.go @@ -457,7 +457,7 @@ func getCanonicalRoot(ctx context.Context, ns NodeStore, builder NodeBuilder) (N assertTrue(cnt == 1) nd := builder.Build(ns.Pool()) - mt := nd.getChildAddress(0) + mt := nd.getAddress(0) for { child, err := fetchChild(ctx, ns, mt) @@ -469,6 +469,6 @@ func getCanonicalRoot(ctx context.Context, ns NodeStore, builder NodeBuilder) (N return child, nil } - mt = child.getChildAddress(0) + mt = child.getAddress(0) } } diff --git a/go/store/prolly/tree/content_address_test.go b/go/store/prolly/tree/content_address_test.go index 35acf77019..eb33b80a39 100644 --- a/go/store/prolly/tree/content_address_test.go +++ b/go/store/prolly/tree/content_address_test.go @@ -26,10 +26,10 @@ import ( ) var goldenHash = hash.Hash{ - 0x9c, 0xf1, 0x55, 0xce, 0x1c, - 0xfc, 0x6e, 0xd5, 0x4b, 0xa9, - 0xcc, 0x7e, 0x15, 0x93, 0xa8, - 0x29, 0x47, 0x16, 0xa8, 0x89, + 0xab, 0x85, 0xcf, 0x59, 0x49, + 0x5d, 0x32, 0x8, 0xbf, 0x63, + 0xad, 0xc3, 0x78, 0x80, 0xc6, + 0xa2, 0xd4, 0x7e, 0x40, 0xae, } // todo(andy): need and analogous test in pkg prolly diff --git a/go/store/prolly/tree/node.go b/go/store/prolly/tree/node.go index 5cbdac236e..abf401ea47 100644 --- a/go/store/prolly/tree/node.go +++ b/go/store/prolly/tree/node.go @@ -133,7 +133,7 @@ func (nd Node) Level() int { // IsLeaf returns whether this node is a leaf func (nd Node) IsLeaf() bool { - return int(nd.msg.TreeLevel()) == 0 + return nd.Level() == 0 } // GetKey returns the |ith| key of this node @@ -141,30 +141,25 @@ func (nd Node) GetKey(i int) Item { return nd.keys.GetSlice(i) } -// getValue returns the |ith| value of this node. Only Valid for leaf nodes. +// getValue returns the |ith| value of this node. func (nd Node) getValue(i int) Item { // todo(andy): abstract value access if nd.values.Buf != nil { return nd.values.GetSlice(i) } else { - r := nd.getChildAddress(i) + r := nd.getAddress(i) return r[:] } } -// getChildAddress returns the |ith| address in this node -func (nd Node) getChildAddress(i int) hash.Hash { +// getAddress returns the |ith| address of this node. +// This method assumes values are 20-byte address hashes. +func (nd Node) getAddress(i int) hash.Hash { refs := nd.msg.AddressArrayBytes() start, stop := i*addrSz, (i+1)*addrSz return hash.New(refs[start:stop]) } -// getValueAddress returns the |ith| value address in this node -func (nd Node) getValueAddress(i int) hash.Hash { - o := nd.msg.ValueAddressOffsets(i) - return hash.New(nd.values.Buf[o : o+addrSz]) -} - func (nd Node) getSubtreeCounts() SubtreeCounts { arr := nd.msg.SubtreeCountsBytes() return readSubtreeCounts(int(nd.count), arr) @@ -182,19 +177,19 @@ func walkAddresses(ctx context.Context, nd Node, cb AddressCb) (err error) { arr := nd.msg.AddressArrayBytes() cnt := len(arr) / addrSz for i := 0; i < cnt; i++ { - if err = cb(ctx, nd.getChildAddress(i)); err != nil { + if err = cb(ctx, nd.getAddress(i)); err != nil { return err } } cnt2 := nd.msg.ValueAddressOffsetsLength() for i := 0; i < cnt2; i++ { - if err = cb(ctx, nd.getValueAddress(i)); err != nil { + o := nd.msg.ValueAddressOffsets(i) + addr := hash.New(nd.values.Buf[o : o+addrSz]) + if err = cb(ctx, addr); err != nil { return err } } - - assertFalse((cnt > 0) && (cnt2 > 0)) return } @@ -250,7 +245,7 @@ func OutputProllyNode(w io.Writer, node Node) error { w.Write([]byte(" }")) } else { - ref := node.getChildAddress(i) + ref := hash.New(node.getValue(i)) w.Write([]byte(" ref: #")) w.Write([]byte(ref.String())) diff --git a/go/store/prolly/tree/node_cursor.go b/go/store/prolly/tree/node_cursor.go index c84ab3da5c..301a4d4c54 100644 --- a/go/store/prolly/tree/node_cursor.go +++ b/go/store/prolly/tree/node_cursor.go @@ -219,7 +219,7 @@ func (cur *Cursor) CurrentValue() Item { } func (cur *Cursor) CurrentRef() hash.Hash { - return cur.nd.getChildAddress(cur.idx) + return cur.nd.getAddress(cur.idx) } func (cur *Cursor) currentSubtreeSize() uint64 { @@ -489,9 +489,3 @@ func assertTrue(b bool) { panic("assertion failed") } } - -func assertFalse(b bool) { - if b { - panic("assertion failed") - } -} diff --git a/go/store/prolly/tree/node_test.go b/go/store/prolly/tree/node_test.go index 6800606e75..ca8bc4b853 100644 --- a/go/store/prolly/tree/node_test.go +++ b/go/store/prolly/tree/node_test.go @@ -62,26 +62,6 @@ func TestRoundTripNodeItems(t *testing.T) { } } -func TestGetKeyValueOffsetsVectors(t *testing.T) { - for trial := 0; trial < 100; trial++ { - keys, values := randomNodeItemPairs(t, (rand.Int()%101)+50) - require.True(t, sumSize(keys)+sumSize(values) < MaxVectorOffset) - nd := newLeafNode(keys, values) - - ko1, vo1 := offsetsFromSlicedBuffers(nd.keys, nd.values) - ko2, vo2 := offsetsFromFlatbuffer(nd.msg) - - assert.Equal(t, len(ko1), len(ko2)) - assert.Equal(t, len(ko1), len(keys)-1) - assert.Equal(t, ko1, ko2) - - assert.Equal(t, len(vo1), len(vo2)) - assert.Equal(t, len(vo1), len(values)-1) - assert.Equal(t, vo1, vo2) - - } -} - func TestNodeSize(t *testing.T) { sz := unsafe.Sizeof(Node{}) assert.Equal(t, 136, int(sz)) diff --git a/go/store/prolly/tree/testutils.go b/go/store/prolly/tree/testutils.go index 8617ccb14b..f0253f109e 100644 --- a/go/store/prolly/tree/testutils.go +++ b/go/store/prolly/tree/testutils.go @@ -307,7 +307,7 @@ func (tb *testBuilder) Build(pool pool.BuffPool) (node Node) { serial.ProllyTreeNodeAddKeyType(b, serial.ItemTypeTupleFormatAlpha) serial.ProllyTreeNodeAddValueType(b, serial.ItemTypeTupleFormatAlpha) serial.ProllyTreeNodeAddTreeLevel(b, uint8(tb.level)) - b.Finish(serial.ProllyTreeNodeEnd(b)) + b.FinishWithFileIdentifier(serial.ProllyTreeNodeEnd(b), []byte(serial.ProllyTreeNodeFileID)) tb.reset() buf := b.FinishedBytes()