mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-02-24 00:59:41 -06:00
Code Issues Packages Projects Releases 136 Wiki Activity

added column type, reduced prolly node size

Browse Source
This commit is contained in:
Andy Arthur
2022-01-04 10:01:34 -08:00
parent 39a35e27d1
commit 2e2cc85709
5 changed files with 174 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
go/gen/fb/serial/prolly.go
View File

@@ -22,7 +22,7 @@ import (
flatbuffers "github.com/google/flatbuffers/go"
)
type TupleFormat uint16
type TupleFormat byte
const (
TupleFormatUnknown TupleFormat = 0
@@ -230,41 +230,29 @@ func (rcv *Map) MutateRefArray(j int, n byte) bool {
func (rcv *Map) KeyFormat() TupleFormat {
o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
if o != 0 {
return TupleFormat(rcv._tab.GetUint16(o + rcv._tab.Pos))
return TupleFormat(rcv._tab.GetByte(o + rcv._tab.Pos))
}
return 0
}
func (rcv *Map) MutateKeyFormat(n TupleFormat) bool {
return rcv._tab.MutateUint16Slot(14, uint16(n))
return rcv._tab.MutateByteSlot(14, byte(n))
}
func (rcv *Map) ValueFormat() TupleFormat {
o := flatbuffers.UOffsetT(rcv._tab.Offset(16))
if o != 0 {
return TupleFormat(rcv._tab.GetUint16(o + rcv._tab.Pos))
return TupleFormat(rcv._tab.GetByte(o + rcv._tab.Pos))
}
return 0
}
func (rcv *Map) MutateValueFormat(n TupleFormat) bool {
return rcv._tab.MutateUint16Slot(16, uint16(n))
}
func (rcv *Map) NodeCount() uint16 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(18))
if o != 0 {
return rcv._tab.GetUint16(o + rcv._tab.Pos)
}
return 0
}
func (rcv *Map) MutateNodeCount(n uint16) bool {
return rcv._tab.MutateUint16Slot(18, n)
return rcv._tab.MutateByteSlot(16, byte(n))
}
func (rcv *Map) TreeCount() uint64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(20))
o := flatbuffers.UOffsetT(rcv._tab.Offset(18))
if o != 0 {
return rcv._tab.GetUint64(o + rcv._tab.Pos)
}
@@ -272,11 +260,11 @@ func (rcv *Map) TreeCount() uint64 {
}
func (rcv *Map) MutateTreeCount(n uint64) bool {
return rcv._tab.MutateUint64Slot(20, n)
return rcv._tab.MutateUint64Slot(18, n)
}
func (rcv *Map) TreeLevel() byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(22))
o := flatbuffers.UOffsetT(rcv._tab.Offset(20))
if o != 0 {
return rcv._tab.GetByte(o + rcv._tab.Pos)
}
@@ -284,11 +272,11 @@ func (rcv *Map) TreeLevel() byte {
}
func (rcv *Map) MutateTreeLevel(n byte) bool {
return rcv._tab.MutateByteSlot(22, n)
return rcv._tab.MutateByteSlot(20, n)
}
func MapStart(builder *flatbuffers.Builder) {
builder.StartObject(10)
builder.StartObject(9)
}
func MapAddKeyTuples(builder *flatbuffers.Builder, keyTuples flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(keyTuples), 0)
@@ -321,19 +309,16 @@ func MapStartRefArrayVector(builder *flatbuffers.Builder, numElems int) flatbuff
return builder.StartVector(1, numElems, 1)
}
func MapAddKeyFormat(builder *flatbuffers.Builder, keyFormat TupleFormat) {
builder.PrependUint16Slot(5, uint16(keyFormat), 0)
builder.PrependByteSlot(5, byte(keyFormat), 0)
}
func MapAddValueFormat(builder *flatbuffers.Builder, valueFormat TupleFormat) {
builder.PrependUint16Slot(6, uint16(valueFormat), 0)
}
func MapAddNodeCount(builder *flatbuffers.Builder, nodeCount uint16) {
builder.PrependUint16Slot(7, nodeCount, 0)
builder.PrependByteSlot(6, byte(valueFormat), 0)
}
func MapAddTreeCount(builder *flatbuffers.Builder, treeCount uint64) {
builder.PrependUint64Slot(8, treeCount, 0)
builder.PrependUint64Slot(7, treeCount, 0)
}
func MapAddTreeLevel(builder *flatbuffers.Builder, treeLevel byte) {
builder.PrependByteSlot(9, treeLevel, 0)
builder.PrependByteSlot(8, treeLevel, 0)
}
func MapEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()

151
go/gen/fb/serial/schema.go
View File

@@ -114,19 +114,20 @@ func (rcv *Column) MutateSchemaOrder(n uint16) bool {
return rcv._tab.MutateUint16Slot(8, n)
}
func (rcv *Column) Nullable() bool {
func (rcv *Column) Type(obj *Type) *Type {
o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(Type)
}
obj.Init(rcv._tab.Bytes, x)
return obj
}
return false
return nil
}
func (rcv *Column) MutateNullable(n bool) bool {
return rcv._tab.MutateBoolSlot(10, n)
}
func (rcv *Column) PrimaryKey() bool {
func (rcv *Column) Nullable() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
@@ -134,11 +135,11 @@ func (rcv *Column) PrimaryKey() bool {
return false
}
func (rcv *Column) MutatePrimaryKey(n bool) bool {
func (rcv *Column) MutateNullable(n bool) bool {
return rcv._tab.MutateBoolSlot(12, n)
}
func (rcv *Column) AutoIncrement() bool {
func (rcv *Column) PrimaryKey() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
@@ -146,12 +147,24 @@ func (rcv *Column) AutoIncrement() bool {
return false
}
func (rcv *Column) MutateAutoIncrement(n bool) bool {
func (rcv *Column) MutatePrimaryKey(n bool) bool {
return rcv._tab.MutateBoolSlot(14, n)
}
func (rcv *Column) Default(obj *ColumnDefault) *ColumnDefault {
func (rcv *Column) AutoIncrement() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(16))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
}
return false
}
func (rcv *Column) MutateAutoIncrement(n bool) bool {
return rcv._tab.MutateBoolSlot(16, n)
}
func (rcv *Column) Default(obj *ColumnDefault) *ColumnDefault {
o := flatbuffers.UOffsetT(rcv._tab.Offset(18))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
@@ -164,7 +177,7 @@ func (rcv *Column) Default(obj *ColumnDefault) *ColumnDefault {
}
func (rcv *Column) Constraints(obj *ColumnConstraint, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(18))
o := flatbuffers.UOffsetT(rcv._tab.Offset(20))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 4
@@ -176,7 +189,7 @@ func (rcv *Column) Constraints(obj *ColumnConstraint, j int) bool {
}
func (rcv *Column) ConstraintsLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(18))
o := flatbuffers.UOffsetT(rcv._tab.Offset(20))
if o != 0 {
return rcv._tab.VectorLen(o)
}
@@ -184,7 +197,7 @@ func (rcv *Column) ConstraintsLength() int {
}
func (rcv *Column) Comment() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(20))
o := flatbuffers.UOffsetT(rcv._tab.Offset(22))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
@@ -192,7 +205,7 @@ func (rcv *Column) Comment() []byte {
}
func ColumnStart(builder *flatbuffers.Builder) {
builder.StartObject(9)
builder.StartObject(10)
}
func ColumnAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(name), 0)
@@ -203,31 +216,125 @@ func ColumnAddStorageOrder(builder *flatbuffers.Builder, storageOrder uint16) {
func ColumnAddSchemaOrder(builder *flatbuffers.Builder, schemaOrder uint16) {
builder.PrependUint16Slot(2, schemaOrder, 0)
}
func ColumnAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(type_), 0)
}
func ColumnAddNullable(builder *flatbuffers.Builder, nullable bool) {
builder.PrependBoolSlot(3, nullable, false)
builder.PrependBoolSlot(4, nullable, false)
}
func ColumnAddPrimaryKey(builder *flatbuffers.Builder, primaryKey bool) {
builder.PrependBoolSlot(4, primaryKey, false)
builder.PrependBoolSlot(5, primaryKey, false)
}
func ColumnAddAutoIncrement(builder *flatbuffers.Builder, autoIncrement bool) {
builder.PrependBoolSlot(5, autoIncrement, false)
builder.PrependBoolSlot(6, autoIncrement, false)
}
func ColumnAddDefault(builder *flatbuffers.Builder, default_ flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(6, flatbuffers.UOffsetT(default_), 0)
builder.PrependUOffsetTSlot(7, flatbuffers.UOffsetT(default_), 0)
}
func ColumnAddConstraints(builder *flatbuffers.Builder, constraints flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(7, flatbuffers.UOffsetT(constraints), 0)
builder.PrependUOffsetTSlot(8, flatbuffers.UOffsetT(constraints), 0)
}
func ColumnStartConstraintsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func ColumnAddComment(builder *flatbuffers.Builder, comment flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(8, flatbuffers.UOffsetT(comment), 0)
builder.PrependUOffsetTSlot(9, flatbuffers.UOffsetT(comment), 0)
}
func ColumnEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
type Type struct {
_tab flatbuffers.Table
}
func GetRootAsType(buf []byte, offset flatbuffers.UOffsetT) *Type {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Type{}
x.Init(buf, n+offset)
return x
}
func GetSizePrefixedRootAsType(buf []byte, offset flatbuffers.UOffsetT) *Type {
n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
x := &Type{}
x.Init(buf, n+offset+flatbuffers.SizeUint32)
return x
}
func (rcv *Type) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Type) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *Type) Type() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
func (rcv *Type) ParamKeys(j int) []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.ByteVector(a + flatbuffers.UOffsetT(j*4))
}
return nil
}
func (rcv *Type) ParamKeysLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func (rcv *Type) ParamValues(j int) []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.ByteVector(a + flatbuffers.UOffsetT(j*4))
}
return nil
}
func (rcv *Type) ParamValuesLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func TypeStart(builder *flatbuffers.Builder) {
builder.StartObject(3)
}
func TypeAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(type_), 0)
}
func TypeAddParamKeys(builder *flatbuffers.Builder, paramKeys flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(paramKeys), 0)
}
func TypeStartParamKeysVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func TypeAddParamValues(builder *flatbuffers.Builder, paramValues flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(paramValues), 0)
}
func TypeStartParamValuesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func TypeEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
type ColumnDefault struct {
_tab flatbuffers.Table
}

40
go/performance/serialbench/serial_test.go
View File

@@ -28,30 +28,30 @@ import (
var (
SmallLeafNodeHash = hash.Hash{
0x67, 0x0, 0x19, 0xd7, 0xf,
0xb2, 0xe9, 0x5f, 0x5e, 0xaf,
0xbb, 0x1e, 0x8, 0xed, 0x3,
0x29, 0xd3, 0xf4, 0x1a, 0xb8,
0x48, 0x65, 0xab, 0x77, 0x80,
0xf7, 0xb, 0xac, 0xf2, 0x98,
0x20, 0x77, 0x62, 0x5f, 0x6,
0x2, 0x32, 0xb5, 0xd4, 0x12,
}
LargeLeafNodeHash = hash.Hash{
0x2e, 0x70, 0x35, 0x93, 0x1b,
0xe7, 0x26, 0x80, 0x15, 0xee,
0x2b, 0x31, 0xfe, 0x9a, 0x41,
0x89, 0x6a, 0x2e, 0x5e, 0x9b,
0x38, 0xdb, 0x6, 0x49, 0x3e,
0x2a, 0xa4, 0x73, 0x67, 0xac,
0x55, 0xd9, 0xa7, 0x8c, 0xcd,
0x69, 0xe2, 0x76, 0x1e, 0xa5,
}
)
const (
SmallLeafNodeSz = 232
SmallLeafNodeSz = 224
SmallKeyTuplesSz = 40
SmallValueTuplesSz = 70
// data 3200 = 800 + 3200
// offsets 800 = sizeof(uint16) * (200 + 200)
// metadata 15 = TupleFormat*2, tree_count, node_count, tree_level
// flatbuffer 65 (1.6% overhead)
// total size 4080
LargeLeafNodeSz = 4080
// metadata 11 = TupleFormat * 2, tree_count, tree_level
// fb + pad 57 = 56 + 1 (1.4% overhead)
// total size 4072
LargeLeafNodeSz = 4072
LargeLeafNodeCount = 200
LargeKeyTuplesSz = 800
LargeValueTuplesSz = 2400
@@ -131,19 +131,17 @@ func makeLeafNode(t *testing.T, keys, values [][]byte) []byte {
assert.Equal(t, start+16, int(b.Offset()))
serial.MapAddTreeCount(b, uint64(len(keys)))
assert.Equal(t, start+24, int(b.Offset()))
serial.MapAddNodeCount(b, uint16(len(keys)))
assert.Equal(t, start+26, int(b.Offset()))
serial.MapAddKeyFormat(b, serial.TupleFormatV1)
assert.Equal(t, start+28, int(b.Offset()))
assert.Equal(t, start+25, int(b.Offset()))
serial.MapAddValueFormat(b, serial.TupleFormatV1)
assert.Equal(t, start+30, int(b.Offset()))
assert.Equal(t, start+26, int(b.Offset()))
serial.MapAddTreeLevel(b, 0)
assert.Equal(t, start+30, int(b.Offset()))
assert.Equal(t, start+26, int(b.Offset()))
mapEnd := serial.MapEnd(b)
assert.Equal(t, start+58, int(b.Offset()))
assert.Equal(t, start+52, int(b.Offset()))
b.Finish(mapEnd)
assert.Equal(t, start+64, int(b.Offset()))
assert.Equal(t, start+56, int(b.Offset()))
return b.FinishedBytes()
}
@@ -165,7 +163,7 @@ func validateLeafNode(t *testing.T, flatbuffer []byte, keys, values [][]byte) {
assert.Equal(t, serial.TupleFormatV1, m.KeyFormat())
assert.Equal(t, serial.TupleFormatV1, m.ValueFormat())
assert.Equal(t, len(keys), int(m.TreeCount()))
assert.Equal(t, len(keys), int(m.NodeCount()))
assert.Equal(t, len(keys), m.KeyOffsetsLength())
assert.Equal(t, 0, int(m.TreeLevel()))
}

11
go/serial/prolly.fbs
View File

@@ -14,19 +14,19 @@
namespace serial;
enum TupleFormat : uint16 {
enum TupleFormat : uint8 {
Unknown,
V1 = 1,
}
table Map {
// array of map key tuples, ordered lexigraphically
key_tuples:[uint8] (required);
key_tuples:[ubyte] (required);
// array of offets into |key_tuples|, zeroth offset omitted
key_offsets:[uint16] (required);
// array of map values tuples, ordered by paired key
value_tuples:[uint8];
value_tuples:[ubyte];
// array of offets into |value_tuples|, zeroth offset omitted
value_offsets:[uint16];
@@ -35,16 +35,13 @@ table Map {
// Prolly tree children
// - leaf nodes: contains refs for heap values
// iff table has ref'd types (eg TEXT)
ref_array:[uint8];
ref_array:[ubyte];
// tuple format for |key_tuples|
key_format:TupleFormat;
// tuple format for |value_tuples|
value_format:TupleFormat;
// node key count
node_count:uint16;
// subtree key count
tree_count:uint64;

8
go/serial/schema.fbs
View File

@@ -19,6 +19,7 @@ table Column {
storage_order:uint16;
schema_order:uint16;
type:Type (required);
nullable:bool;
primary_key:bool;
auto_increment:bool;
@@ -27,6 +28,13 @@ table Column {
comment:string (required);
}
// based on schema_marshalling.go:encodeTypeInfo()
table Type {
type:string (required);
param_keys:[string] (required);
param_values:[string] (required);
}
table ColumnDefault {
expression:string (required);
}