mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-02-14 18:18:55 -06:00
Code Issues Packages Projects Releases 132 Wiki Activity

comment doc, cleanup

Browse Source
This commit is contained in:
Andy Arthur
2022-05-03 10:05:01 -07:00
parent 0087c58d8c
commit f49fdf78ee
8 changed files with 18 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go/gen/fb/serial/fileidentifiers.go
View File

@@ -22,7 +22,7 @@ const WorkingSetFileID = "WRST"
const CommitFileID = "DCMT"
const RootValueFileID = "RTVL"
const TableFileID = "DTBL"
const ProllyTreeNodeFileID = "PRLY"
const ProllyTreeNodeFileID = "TUPM"
const AddressMapFileID = "ADRM"
func GetFileID(bs []byte) string {

55
go/gen/fb/serial/prolly.go
View File

@@ -396,6 +396,7 @@ func ProllyTreeNodeEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
/// Refmap has been deprecated in favor of AddressMap
type RefMap struct {
_tab flatbuffers.Table
}
@@ -474,42 +475,8 @@ func (rcv *RefMap) MutateRefArray(j int, n byte) bool {
return false
}
func (rcv *RefMap) SubtreeCounts(j int) byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1))
}
return 0
}
func (rcv *RefMap) SubtreeCountsLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func (rcv *RefMap) SubtreeCountsBytes() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
func (rcv *RefMap) MutateSubtreeCounts(j int, n byte) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.MutateByte(a+flatbuffers.UOffsetT(j*1), n)
}
return false
}
func (rcv *RefMap) TreeCount() uint64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.GetUint64(o + rcv._tab.Pos)
}
@@ -517,11 +484,11 @@ func (rcv *RefMap) TreeCount() uint64 {
}
func (rcv *RefMap) MutateTreeCount(n uint64) bool {
return rcv._tab.MutateUint64Slot(10, n)
return rcv._tab.MutateUint64Slot(8, n)
}
func (rcv *RefMap) TreeLevel() byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
if o != 0 {
return rcv._tab.GetByte(o + rcv._tab.Pos)
}
@@ -529,11 +496,11 @@ func (rcv *RefMap) TreeLevel() byte {
}
func (rcv *RefMap) MutateTreeLevel(n byte) bool {
return rcv._tab.MutateByteSlot(12, n)
return rcv._tab.MutateByteSlot(10, n)
}
func RefMapStart(builder *flatbuffers.Builder) {
builder.StartObject(5)
builder.StartObject(4)
}
func RefMapAddNames(builder *flatbuffers.Builder, names flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(names), 0)
@@ -547,17 +514,11 @@ func RefMapAddRefArray(builder *flatbuffers.Builder, refArray flatbuffers.UOffse
func RefMapStartRefArrayVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(1, numElems, 1)
}
func RefMapAddSubtreeCounts(builder *flatbuffers.Builder, subtreeCounts flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(subtreeCounts), 0)
}
func RefMapStartSubtreeCountsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(1, numElems, 1)
}
func RefMapAddTreeCount(builder *flatbuffers.Builder, treeCount uint64) {
builder.PrependUint64Slot(3, treeCount, 0)
builder.PrependUint64Slot(2, treeCount, 0)
}
func RefMapAddTreeLevel(builder *flatbuffers.Builder, treeLevel byte) {
builder.PrependByteSlot(4, treeLevel, 0)
builder.PrependByteSlot(3, treeLevel, 0)
}
func RefMapEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()

2
go/serial/fileidentifiers.go
View File

@@ -22,7 +22,7 @@ const WorkingSetFileID = "WRST"
const CommitFileID = "DCMT"
const RootValueFileID = "RTVL"
const TableFileID = "DTBL"
const ProllyTreeNodeFileID = "PRLY"
const ProllyTreeNodeFileID = "TUPM"
const AddressMapFileID = "ADRM"
func GetFileID(bs []byte) string {

7
go/serial/prolly.fbs
View File

@@ -52,7 +52,7 @@ table ProllyTreeNode {
tree_level:uint8;
}
// todo(andy): rename to AddressMap
/// Refmap has been deprecated in favor of AddressMap
table RefMap {
// map keys ordered lexigraphically
names:[string] (required);
@@ -60,11 +60,8 @@ table RefMap {
// array of chunk addresses
// - subtree addresses for internal prolly tree nodes
// - value addresses for AddressMap leaf nodes
// todo(andy): rename to address_array
ref_array:[ubyte] (required);
// array of uvarint encoded subtree counts
subtree_counts:[ubyte];
// total count of prolly tree
tree_count:uint64;
// prolly tree level, 0 for leaf nodes
@@ -84,6 +81,6 @@ table CommitClosure {
// KEEP THIS IN SYNC WITH fileidentifiers.go
file_identifier "PRLY";
file_identifier "TUPM";
root_type ProllyTreeNode;

5
go/store/prolly/message/addresses.go
View File

@@ -42,6 +42,11 @@ func init() {
}
}
// offsetsForAddressArray provides an uint16 offsets array |offs| for an array
// of addresses |arr|. Together, |arr| and |offs| can construct a val.SlicedBuffer.
// Offsets aren't necessary to slice into an array of fixed-width addresses, but
// we still wrap address arrays in val.SlicedBuffer to provide a uniform API when
// accessing keys and values of Messages.
func offsetsForAddressArray(arr []byte) (offs []byte) {
cnt := len(arr) / addrSize
offs = addressOffsets[:cnt*uint16Size]

90
go/store/prolly/serialize.go
View File

@@ -1,90 +0,0 @@
// Copyright 2021 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prolly
import (
"encoding/binary"
"fmt"
fb "github.com/google/flatbuffers/go"
"github.com/dolthub/dolt/go/store/pool"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/prolly/tree"
)
func getFlatbufferBuilder(pool pool.BuffPool, sz int) (b *fb.Builder) {
b = fb.NewBuilder(0)
buf := pool.Get(uint64(sz))
b.Bytes = buf[:0]
return
}
func writeItemBytes(b *fb.Builder, items []tree.Item, sumSz int) fb.UOffsetT {
b.Prep(fb.SizeUOffsetT, sumSz)
stop := int(b.Head())
start := stop - sumSz
for _, item := range items {
copy(b.Bytes[start:stop], item)
start += len(item)
}
start = stop - sumSz
return b.CreateByteVector(b.Bytes[start:stop])
}
func writeItemOffsets(b *fb.Builder, items []tree.Item, sumSz int) fb.UOffsetT {
var cnt int
var off = sumSz
for i := len(items) - 1; i > 0; i-- { // omit first offset
off -= len(items[i])
b.PrependUint16(uint16(off))
cnt++
}
return b.EndVector(cnt)
}
func writeCountArray(b *fb.Builder, sc tree.SubtreeCounts) fb.UOffsetT {
// todo(andy) write without copy
arr := message.WriteSubtreeCounts(sc)
return b.CreateByteVector(arr)
}
// estimateBufferSize returns the exact Size of the tuple vectors for keys and values,
// and an estimate of the overall Size of the final flatbuffer.
func estimateBufferSize(keys, values []tree.Item, subtrees []uint64) (keySz, valSz, bufSz int) {
for i := range keys {
keySz += len(keys[i])
valSz += len(values[i])
}
refCntSz := len(subtrees) * binary.MaxVarintLen64
// constraints enforced upstream
if keySz > int(message.MaxVectorOffset) {
panic(fmt.Sprintf("key vector exceeds Size limit ( %d > %d )", keySz, message.MaxVectorOffset))
}
if valSz > int(message.MaxVectorOffset) {
panic(fmt.Sprintf("value vector exceeds Size limit ( %d > %d )", valSz, message.MaxVectorOffset))
}
bufSz += keySz + valSz // tuples
bufSz += refCntSz // subtree counts
bufSz += len(keys)*2 + len(values)*2 // offsets
bufSz += 8 + 1 + 1 + 1 // metadata
bufSz += 72 // vtable (approx)
return
}

10
go/store/prolly/tree/node.go
View File

@@ -25,14 +25,6 @@ import (
"github.com/dolthub/dolt/go/store/val"
)
const (
// These constants are mirrored from serial.ProllyTreeNode.KeyOffsetsLength()
// and serial.ProllyTreeNode.ValueOffsetsLength() respectively.
// They are only as stable as the flatbuffers schemas that define them.
keyOffsetsVOffset = 6
valueOffsetsVOffset = 12
)
type Item []byte
type Node struct {
@@ -175,7 +167,7 @@ func OutputProllyNode(w io.Writer, node Node) error {
w.Write([]byte(" }"))
} else {
ref := hash.New(node.getValue(i))
ref := node.getAddress(i)
w.Write([]byte(" ref: #"))
w.Write([]byte(ref.String()))

2
go/store/prolly/tree/testutils.go
View File

@@ -155,8 +155,6 @@ func ShuffleTuplePairs(items [][2]val.Tuple) {
}
func newLeafNode(keys, values []Item) Node {
// furp
kk := make([][]byte, len(keys))
for i := range keys {
kk[i] = keys[i]