From 21f15163744a686d1a722b8d98820a8e9e934400 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 23 Jan 2023 16:07:52 -0800 Subject: [PATCH 01/15] starting of new enc --- go/store/val/codec.go | 10 ++++++++++ go/store/val/codec_test.go | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/go/store/val/codec.go b/go/store/val/codec.go index 9cb83c3813..5f0a7a762d 100644 --- a/go/store/val/codec.go +++ b/go/store/val/codec.go @@ -63,6 +63,7 @@ const ( commitAddrEnc ByteSize = hash.ByteLen stringAddrEnc ByteSize = hash.ByteLen jsonAddrEnc ByteSize = hash.ByteLen + zAddrSize ByteSize = 17 ) type Encoding byte @@ -597,6 +598,10 @@ func compareAddr(l, r hash.Hash) int { return l.Compare(r) } +func compareZAddr(l, r []byte) int { + return bytes.Compare(l, r) +} + func writeRaw(buf, val []byte) { expectSize(buf, ByteSize(len(val))) copy(buf, val) @@ -622,3 +627,8 @@ func expectSize(buf []byte, sz ByteSize) { func stringFromBytes(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } + +func readZAddr(val []byte) []byte { + expectSize(val, zAddrSize) + return val +} diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index 632a13ce69..606b729785 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -240,6 +240,12 @@ func TestCompare(t *testing.T) { l: encStr("b"), r: encStr("a"), cmp: 1, }, + // z-address / geometry + { + typ: Type{Enc: GeometryEnc}, + l: encStr("b"), r: encStr("a"), + cmp: 1, + }, } for _, test := range tests { From 495b487b58f7eb7b7a23b56870d967cd90f81f76 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 23 Jan 2023 16:58:37 -0800 Subject: [PATCH 02/15] adding encoding --- go/serial/encoding.fbs | 1 + go/store/val/codec.go | 1 + 2 files changed, 2 insertions(+) diff --git a/go/serial/encoding.fbs b/go/serial/encoding.fbs index 07f7cc6352..7928413fe3 100644 --- a/go/serial/encoding.fbs +++ b/go/serial/encoding.fbs @@ -39,6 +39,7 @@ enum Encoding : uint8 { CommitAddr = 22, StringAddr = 23, JSONAddr = 24, + ZAddr = 25, // variable width String = 128, diff --git a/go/store/val/codec.go b/go/store/val/codec.go index 5f0a7a762d..9a58fa374d 100644 --- a/go/store/val/codec.go +++ b/go/store/val/codec.go @@ -93,6 +93,7 @@ const ( CommitAddrEnc = Encoding(serial.EncodingCommitAddr) StringAddrEnc = Encoding(serial.EncodingStringAddr) JSONAddrEnc = Encoding(serial.EncodingJSONAddr) + ZAddrEnc = Encoding(serial.EncodingZAddr) sentinel Encoding = 127 ) From f48047dbc1d847bc133e71655c22ed9b6a3d02cc Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 23 Jan 2023 22:33:18 -0800 Subject: [PATCH 03/15] aa --- go/gen/fb/serial/encoding.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/gen/fb/serial/encoding.go b/go/gen/fb/serial/encoding.go index f5de356322..f9fbc8b32b 100644 --- a/go/gen/fb/serial/encoding.go +++ b/go/gen/fb/serial/encoding.go @@ -46,6 +46,7 @@ const ( EncodingCommitAddr Encoding = 22 EncodingStringAddr Encoding = 23 EncodingJSONAddr Encoding = 24 + EncodingZAddr Encoding = 25 EncodingString Encoding = 128 EncodingBytes Encoding = 129 EncodingDecimal Encoding = 130 @@ -77,6 +78,7 @@ var EnumNamesEncoding = map[Encoding]string{ EncodingCommitAddr: "CommitAddr", EncodingStringAddr: "StringAddr", EncodingJSONAddr: "JSONAddr", + EncodingZAddr: "ZAddr", EncodingString: "String", EncodingBytes: "Bytes", EncodingDecimal: "Decimal", @@ -108,6 +110,7 @@ var EnumValuesEncoding = map[string]Encoding{ "CommitAddr": EncodingCommitAddr, "StringAddr": EncodingStringAddr, "JSONAddr": EncodingJSONAddr, + "ZAddr": EncodingZAddr, "String": EncodingString, "Bytes": EncodingBytes, "Decimal": EncodingDecimal, From 3f98425b46bbee2e6bf4754c7cd103478d2a42e3 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 24 Jan 2023 16:14:05 -0800 Subject: [PATCH 04/15] refactor and create new codec --- go/store/val/codec_test.go | 13 ++++++++++--- go/store/val/tuple_compare.go | 2 ++ .../doltcore/sqle/index => store/val}/z_encoding.go | 2 +- .../sqle/index => store/val}/z_encoding_test.go | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) rename go/{libraries/doltcore/sqle/index => store/val}/z_encoding.go (99%) rename go/{libraries/doltcore/sqle/index => store/val}/z_encoding_test.go (99%) diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index 606b729785..667d836f17 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -15,6 +15,7 @@ package val import ( + "github.com/dolthub/go-mysql-server/sql/types" "math" "testing" "time" @@ -240,10 +241,11 @@ func TestCompare(t *testing.T) { l: encStr("b"), r: encStr("a"), cmp: 1, }, - // z-address / geometry + // z-address { - typ: Type{Enc: GeometryEnc}, - l: encStr("b"), r: encStr("a"), + typ: Type{Enc: ZAddrEnc}, + l: encZAddr(types.Point{X: 9999, Y: 12312312}), + r: encZAddr(types.Point{X: 123, Y: 456}), cmp: 1, }, } @@ -303,6 +305,11 @@ func encStr(s string) []byte { return buf } +func encZAddr(g types.GeometryValue) []byte { + buf := ZAddr(g) + return buf[:] +} + func encYear(y int16) []byte { buf := make([]byte, yearSize) writeYear(buf, y) diff --git a/go/store/val/tuple_compare.go b/go/store/val/tuple_compare.go index 8e22afa722..6c524f7680 100644 --- a/go/store/val/tuple_compare.go +++ b/go/store/val/tuple_compare.go @@ -143,6 +143,8 @@ func compare(typ Type, left, right []byte) int { return compareAddr(readAddr(left), readAddr(right)) case StringAddrEnc: return compareAddr(readAddr(left), readAddr(right)) + case ZAddrEnc: + return compareZAddr(readZAddr(left), readZAddr(right)) default: panic("unknown encoding") } diff --git a/go/libraries/doltcore/sqle/index/z_encoding.go b/go/store/val/z_encoding.go similarity index 99% rename from go/libraries/doltcore/sqle/index/z_encoding.go rename to go/store/val/z_encoding.go index 33efd44d8a..a082e0736d 100644 --- a/go/libraries/doltcore/sqle/index/z_encoding.go +++ b/go/store/val/z_encoding.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package index +package val import ( "bytes" diff --git a/go/libraries/doltcore/sqle/index/z_encoding_test.go b/go/store/val/z_encoding_test.go similarity index 99% rename from go/libraries/doltcore/sqle/index/z_encoding_test.go rename to go/store/val/z_encoding_test.go index 48e446825f..c80e7c7ce4 100644 --- a/go/libraries/doltcore/sqle/index/z_encoding_test.go +++ b/go/store/val/z_encoding_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package index +package val import ( "encoding/hex" From ed3352ca5d432dcd8ef41d405aed08e70b65f7ed Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 24 Jan 2023 16:16:50 -0800 Subject: [PATCH 05/15] more tests --- go/store/val/codec_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index 667d836f17..46c66c4861 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -248,6 +248,18 @@ func TestCompare(t *testing.T) { r: encZAddr(types.Point{X: 123, Y: 456}), cmp: 1, }, + { + typ: Type{Enc: ZAddrEnc}, + r: encZAddr(types.Point{}), + l: encZAddr(types.LineString{Points: []types.Point{{}, {}, {}}}), + cmp: 0, + }, + { + typ: Type{Enc: ZAddrEnc}, + r: encZAddr(types.Point{X: 123, Y: 456}), + l: encZAddr(types.LineString{Points: []types.Point{{X: -10, Y: -10}, {X: 10, Y: 10}}}), + cmp: -1, + }, } for _, test := range tests { From 4dfb647d2fef6c0172e738ea9e406c976a0b4517 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 24 Jan 2023 16:55:12 -0800 Subject: [PATCH 06/15] move back --- .../doltcore/sqle/index}/z_encoding.go | 2 +- .../doltcore/sqle/index}/z_encoding_test.go | 2 +- go/store/val/codec.go | 9 +++++---- go/store/val/codec_test.go | 16 ---------------- 4 files changed, 7 insertions(+), 22 deletions(-) rename go/{store/val => libraries/doltcore/sqle/index}/z_encoding.go (99%) rename go/{store/val => libraries/doltcore/sqle/index}/z_encoding_test.go (99%) diff --git a/go/store/val/z_encoding.go b/go/libraries/doltcore/sqle/index/z_encoding.go similarity index 99% rename from go/store/val/z_encoding.go rename to go/libraries/doltcore/sqle/index/z_encoding.go index a082e0736d..33efd44d8a 100644 --- a/go/store/val/z_encoding.go +++ b/go/libraries/doltcore/sqle/index/z_encoding.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package val +package index import ( "bytes" diff --git a/go/store/val/z_encoding_test.go b/go/libraries/doltcore/sqle/index/z_encoding_test.go similarity index 99% rename from go/store/val/z_encoding_test.go rename to go/libraries/doltcore/sqle/index/z_encoding_test.go index c80e7c7ce4..48e446825f 100644 --- a/go/store/val/z_encoding_test.go +++ b/go/libraries/doltcore/sqle/index/z_encoding_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package val +package index import ( "encoding/hex" diff --git a/go/store/val/codec.go b/go/store/val/codec.go index 9a58fa374d..7535251ba1 100644 --- a/go/store/val/codec.go +++ b/go/store/val/codec.go @@ -599,8 +599,8 @@ func compareAddr(l, r hash.Hash) int { return l.Compare(r) } -func compareZAddr(l, r []byte) int { - return bytes.Compare(l, r) +func compareZAddr(l, r [zAddrSize]byte) int { + return bytes.Compare(l[:], r[:]) } func writeRaw(buf, val []byte) { @@ -629,7 +629,8 @@ func stringFromBytes(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } -func readZAddr(val []byte) []byte { +func readZAddr(val []byte) (res [zAddrSize]byte) { expectSize(val, zAddrSize) - return val + copy(res[:zAddrSize], val[:zAddrSize]) + return } diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index 46c66c4861..760ef04c29 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -243,22 +243,6 @@ func TestCompare(t *testing.T) { }, // z-address { - typ: Type{Enc: ZAddrEnc}, - l: encZAddr(types.Point{X: 9999, Y: 12312312}), - r: encZAddr(types.Point{X: 123, Y: 456}), - cmp: 1, - }, - { - typ: Type{Enc: ZAddrEnc}, - r: encZAddr(types.Point{}), - l: encZAddr(types.LineString{Points: []types.Point{{}, {}, {}}}), - cmp: 0, - }, - { - typ: Type{Enc: ZAddrEnc}, - r: encZAddr(types.Point{X: 123, Y: 456}), - l: encZAddr(types.LineString{Points: []types.Point{{X: -10, Y: -10}, {X: 10, Y: 10}}}), - cmp: -1, }, } From d438c6702ac0139415ae89cad3abdfb8644979d7 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 24 Jan 2023 23:22:26 -0800 Subject: [PATCH 07/15] implementing codec --- go/store/val/codec.go | 13 +++++++++---- go/store/val/codec_test.go | 12 ++++++++---- go/store/val/tuple_builder.go | 9 +++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/go/store/val/codec.go b/go/store/val/codec.go index 7535251ba1..3ee6715289 100644 --- a/go/store/val/codec.go +++ b/go/store/val/codec.go @@ -599,10 +599,6 @@ func compareAddr(l, r hash.Hash) int { return l.Compare(r) } -func compareZAddr(l, r [zAddrSize]byte) int { - return bytes.Compare(l[:], r[:]) -} - func writeRaw(buf, val []byte) { expectSize(buf, ByteSize(len(val))) copy(buf, val) @@ -629,8 +625,17 @@ func stringFromBytes(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } +func compareZAddr(l, r [zAddrSize]byte) int { + return bytes.Compare(l[:], r[:]) +} + func readZAddr(val []byte) (res [zAddrSize]byte) { expectSize(val, zAddrSize) copy(res[:zAddrSize], val[:zAddrSize]) return } + +func writeZAddr(buf []byte, v [zAddrSize]byte) { + expectSize(buf, zAddrSize) + copy(buf, v[:zAddrSize]) +} diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index 760ef04c29..6fef577d7b 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -15,7 +15,6 @@ package val import ( - "github.com/dolthub/go-mysql-server/sql/types" "math" "testing" "time" @@ -243,6 +242,10 @@ func TestCompare(t *testing.T) { }, // z-address { + typ: Type{Enc: StringEnc}, + l: encZAddr([zAddrSize]byte{}), + r: encZAddr([zAddrSize]byte{}), + cmp: 0, }, } @@ -301,9 +304,10 @@ func encStr(s string) []byte { return buf } -func encZAddr(g types.GeometryValue) []byte { - buf := ZAddr(g) - return buf[:] +func encZAddr(z [zAddrSize]byte) []byte { + buf := make([]byte, zAddrSize) + writeZAddr(buf, z) + return buf } func encYear(y int16) []byte { diff --git a/go/store/val/tuple_builder.go b/go/store/val/tuple_builder.go index 5a0bcc4af7..40afcdc40b 100644 --- a/go/store/val/tuple_builder.go +++ b/go/store/val/tuple_builder.go @@ -366,3 +366,12 @@ func (tb *TupleBuilder) ensureCapacity(sz ByteSize) { } } } + +// PutZAddr writes a bounding box to the ith field of the Tuple being built. +func (tb *TupleBuilder) PutZAddr(i int, v [zAddrSize]byte) { + tb.Desc.expectEncoding(i, ZAddrEnc) + tb.ensureCapacity(zAddrSize) + tb.fields[i] = tb.buf[tb.pos : tb.pos+zAddrSize] + writeZAddr(tb.fields[i], v) + tb.pos += zAddrSize +} From 26a9ac7b572a696e53f9cb1bd5fb53bf62dbdc59 Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Wed, 25 Jan 2023 09:16:41 +0000 Subject: [PATCH 08/15] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/store/val/codec.go | 2 +- go/store/val/codec_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/store/val/codec.go b/go/store/val/codec.go index 3ee6715289..e68fe21ace 100644 --- a/go/store/val/codec.go +++ b/go/store/val/codec.go @@ -63,7 +63,7 @@ const ( commitAddrEnc ByteSize = hash.ByteLen stringAddrEnc ByteSize = hash.ByteLen jsonAddrEnc ByteSize = hash.ByteLen - zAddrSize ByteSize = 17 + zAddrSize ByteSize = 17 ) type Encoding byte diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index 6fef577d7b..4f58ccc72c 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -243,8 +243,8 @@ func TestCompare(t *testing.T) { // z-address { typ: Type{Enc: StringEnc}, - l: encZAddr([zAddrSize]byte{}), - r: encZAddr([zAddrSize]byte{}), + l: encZAddr([zAddrSize]byte{}), + r: encZAddr([zAddrSize]byte{}), cmp: 0, }, } From 668513b97a17fc70dbebb7b466a3790142f021aa Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 25 Jan 2023 11:41:55 -0800 Subject: [PATCH 09/15] implement tuple builder and descriptor --- go/libraries/doltcore/sqle/index/prolly_fields.go | 4 ++++ go/libraries/doltcore/sqle/index/z_encoding.go | 2 +- go/store/val/tuple_builder.go | 2 +- go/store/val/tuple_descriptor.go | 10 ++++++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/index/prolly_fields.go b/go/libraries/doltcore/sqle/index/prolly_fields.go index 82b53dfdc3..772077b96d 100644 --- a/go/libraries/doltcore/sqle/index/prolly_fields.go +++ b/go/libraries/doltcore/sqle/index/prolly_fields.go @@ -118,6 +118,8 @@ func GetField(ctx context.Context, td val.TupleDesc, i int, tup val.Tuple, ns tr } case val.CommitAddrEnc: v, ok = td.GetCommitAddr(i, tup) + case val.ZAddrEnc: + v, ok = td.GetZAddr(i, tup) default: panic("unknown val.encoding") } @@ -214,6 +216,8 @@ func PutField(ctx context.Context, ns tree.NodeStore, tb *val.TupleBuilder, i in tb.PutStringAddr(i, h) case val.CommitAddrEnc: tb.PutCommitAddr(i, v.(hash.Hash)) + case val.ZAddrEnc: + tb.PutZAddr(i, ZAddr(v.(types.GeometryValue))) default: panic(fmt.Sprintf("unknown encoding %v %v", enc, v)) } diff --git a/go/libraries/doltcore/sqle/index/z_encoding.go b/go/libraries/doltcore/sqle/index/z_encoding.go index 33efd44d8a..0264adf2b1 100644 --- a/go/libraries/doltcore/sqle/index/z_encoding.go +++ b/go/libraries/doltcore/sqle/index/z_encoding.go @@ -163,7 +163,7 @@ func ZAddr(v types.GeometryValue) [17]byte { if res := zMin[0] ^ zMax[0]; res != 0 { addr[0] = byte(64 - bits.LeadingZeros64(res)/2) } else { - addr[0] = byte(32 + bits.LeadingZeros64(zMin[1]^zMax[1])/2) + addr[0] = byte(32 - bits.LeadingZeros64(zMin[1]^zMax[1])/2) } return addr } diff --git a/go/store/val/tuple_builder.go b/go/store/val/tuple_builder.go index 40afcdc40b..3c00c15095 100644 --- a/go/store/val/tuple_builder.go +++ b/go/store/val/tuple_builder.go @@ -367,7 +367,7 @@ func (tb *TupleBuilder) ensureCapacity(sz ByteSize) { } } -// PutZAddr writes a bounding box to the ith field of the Tuple being built. +// PutZAddr writes a Z-Address to the ith field of the Tuple being built. func (tb *TupleBuilder) PutZAddr(i int, v [zAddrSize]byte) { tb.Desc.expectEncoding(i, ZAddrEnc) tb.ensureCapacity(zAddrSize) diff --git a/go/store/val/tuple_descriptor.go b/go/store/val/tuple_descriptor.go index 83a2b7cf30..9a2139ff15 100644 --- a/go/store/val/tuple_descriptor.go +++ b/go/store/val/tuple_descriptor.go @@ -468,6 +468,16 @@ func (td TupleDesc) expectEncoding(i int, encodings ...Encoding) { panic("incorrect value encoding") } +func (td TupleDesc) GetZAddr(i int, tup Tuple) (v [zAddrSize]byte, ok bool) { + td.expectEncoding(i, ZAddrEnc) + b := td.GetField(i, tup) + if b != nil { + v = readZAddr(b) + ok = true + } + return +} + // Format prints a Tuple as a string. func (td TupleDesc) Format(tup Tuple) string { if tup == nil || tup.Count() == 0 { From a60577934c9446b5084dba920e89706323e1be07 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 25 Jan 2023 11:46:14 -0800 Subject: [PATCH 10/15] fixing test with bad level encoding --- .../doltcore/sqle/index/z_encoding_test.go | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/go/libraries/doltcore/sqle/index/z_encoding_test.go b/go/libraries/doltcore/sqle/index/z_encoding_test.go index 48e446825f..0bc8daed78 100644 --- a/go/libraries/doltcore/sqle/index/z_encoding_test.go +++ b/go/libraries/doltcore/sqle/index/z_encoding_test.go @@ -197,7 +197,7 @@ func TestZAddr(t *testing.T) { t.Run("test points z-addrs", func(t *testing.T) { p := types.Point{X: 1, Y: 2} res := ZAddr(p) - assert.Equal(t, "40e5555500000000000000000000000000", hex.EncodeToString(res[:])) + assert.Equal(t, "00e5555500000000000000000000000000", hex.EncodeToString(res[:])) }) t.Run("test linestring z-addrs", func(t *testing.T) { @@ -222,35 +222,35 @@ func TestZAddr(t *testing.T) { } func TestZSort(t *testing.T) { - p1 := types.LineString{Points: []types.Point{ps[16], ps[19]}} - p2 := types.LineString{Points: []types.Point{ps[0], ps[3]}} - p3 := types.LineString{Points: []types.Point{ps[19], ps[24]}} - p4 := types.LineString{Points: []types.Point{ps[3], ps[19]}} - p5 := types.LineString{Points: []types.Point{ps[24], ps[24]}} + p1 := types.LineString{Points: []types.Point{ps[24], ps[24]}} + p2 := types.LineString{Points: []types.Point{ps[16], ps[19]}} + p3 := types.LineString{Points: []types.Point{ps[0], ps[3]}} + p4 := types.LineString{Points: []types.Point{ps[19], ps[24]}} + p5 := types.LineString{Points: []types.Point{ps[3], ps[19]}} t.Run("test z-addr p1", func(t *testing.T) { - z := ZAddr(p1) // bbox: (0, 0), (1, 1) - assert.Equal(t, "3ec0000000000000000000000000000000", hex.EncodeToString(z[:])) + z := ZAddr(p1) // bbox: (2, 2), (2, 2) + assert.Equal(t, "00f0000000000000000000000000000000", hex.EncodeToString(z[:])) }) t.Run("test z-addr p2", func(t *testing.T) { - z := ZAddr(p2) // bbox: (-2, -2), (-1, -1) - assert.Equal(t, "3f0fffffffffffffffffffffffffffffff", hex.EncodeToString(z[:])) + z := ZAddr(p2) // bbox: (0, 0), (1, 1) + assert.Equal(t, "3ec0000000000000000000000000000000", hex.EncodeToString(z[:])) }) t.Run("test z-addr p3", func(t *testing.T) { - z := ZAddr(p3) // bbox: (1, 1), (2, 2) - assert.Equal(t, "3fcfffff00000000000000000000000000", hex.EncodeToString(z[:])) + z := ZAddr(p3) // bbox: (-2, -2), (-1, -1) + assert.Equal(t, "3f0fffffffffffffffffffffffffffffff", hex.EncodeToString(z[:])) }) t.Run("test z-addr p4", func(t *testing.T) { - z := ZAddr(p4) // bbox: (-1, -1), (1, 1) - assert.Equal(t, "40300000ffffffffffffffffffffffffff", hex.EncodeToString(z[:])) + z := ZAddr(p4) // bbox: (1, 1), (2, 2) + assert.Equal(t, "3fcfffff00000000000000000000000000", hex.EncodeToString(z[:])) }) - t.Run("test z-addr p6", func(t *testing.T) { - z := ZAddr(p5) // bbox: (2, 2), (2, 2) - assert.Equal(t, "40f0000000000000000000000000000000", hex.EncodeToString(z[:])) + t.Run("test z-addr p5", func(t *testing.T) { + z := ZAddr(p5) // bbox: (-1, -1), (1, 1) + assert.Equal(t, "40300000ffffffffffffffffffffffffff", hex.EncodeToString(z[:])) }) t.Run("test z-addr sorting", func(t *testing.T) { From 7e7b75784b9cc0a7aa7e89107d5b0dd5339da92f Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 25 Jan 2023 13:55:39 -0800 Subject: [PATCH 11/15] zaddr to cell --- go/gen/fb/serial/encoding.go | 10 ++--- .../doltcore/sqle/index/prolly_fields.go | 8 ++-- .../doltcore/sqle/index/z_encoding.go | 28 ++++++++++++-- go/store/val/codec.go | 37 +++++++++++++------ go/store/val/codec_test.go | 10 ++--- go/store/val/tuple_builder.go | 14 +++---- go/store/val/tuple_compare.go | 4 +- go/store/val/tuple_descriptor.go | 8 ++-- 8 files changed, 77 insertions(+), 42 deletions(-) diff --git a/go/gen/fb/serial/encoding.go b/go/gen/fb/serial/encoding.go index f9fbc8b32b..818937b90f 100644 --- a/go/gen/fb/serial/encoding.go +++ b/go/gen/fb/serial/encoding.go @@ -45,9 +45,9 @@ const ( EncodingBytesAddr Encoding = 21 EncodingCommitAddr Encoding = 22 EncodingStringAddr Encoding = 23 - EncodingJSONAddr Encoding = 24 - EncodingZAddr Encoding = 25 - EncodingString Encoding = 128 + EncodingJSONAddr Encoding = 24 + EncodingCell Encoding = 25 + EncodingString Encoding = 128 EncodingBytes Encoding = 129 EncodingDecimal Encoding = 130 EncodingJSON Encoding = 131 @@ -78,7 +78,7 @@ var EnumNamesEncoding = map[Encoding]string{ EncodingCommitAddr: "CommitAddr", EncodingStringAddr: "StringAddr", EncodingJSONAddr: "JSONAddr", - EncodingZAddr: "ZAddr", + EncodingCell: "ZAddr", EncodingString: "String", EncodingBytes: "Bytes", EncodingDecimal: "Decimal", @@ -110,7 +110,7 @@ var EnumValuesEncoding = map[string]Encoding{ "CommitAddr": EncodingCommitAddr, "StringAddr": EncodingStringAddr, "JSONAddr": EncodingJSONAddr, - "ZAddr": EncodingZAddr, + "ZAddr": EncodingCell, "String": EncodingString, "Bytes": EncodingBytes, "Decimal": EncodingDecimal, diff --git a/go/libraries/doltcore/sqle/index/prolly_fields.go b/go/libraries/doltcore/sqle/index/prolly_fields.go index 772077b96d..11b05fba7a 100644 --- a/go/libraries/doltcore/sqle/index/prolly_fields.go +++ b/go/libraries/doltcore/sqle/index/prolly_fields.go @@ -118,8 +118,8 @@ func GetField(ctx context.Context, td val.TupleDesc, i int, tup val.Tuple, ns tr } case val.CommitAddrEnc: v, ok = td.GetCommitAddr(i, tup) - case val.ZAddrEnc: - v, ok = td.GetZAddr(i, tup) + case val.CellEnc: + v, ok = td.GetCell(i, tup) default: panic("unknown val.encoding") } @@ -216,8 +216,8 @@ func PutField(ctx context.Context, ns tree.NodeStore, tb *val.TupleBuilder, i in tb.PutStringAddr(i, h) case val.CommitAddrEnc: tb.PutCommitAddr(i, v.(hash.Hash)) - case val.ZAddrEnc: - tb.PutZAddr(i, ZAddr(v.(types.GeometryValue))) + case val.CellEnc: + tb.PutCell(i, ZCell(v.(types.GeometryValue))) default: panic(fmt.Sprintf("unknown encoding %v %v", enc, v)) } diff --git a/go/libraries/doltcore/sqle/index/z_encoding.go b/go/libraries/doltcore/sqle/index/z_encoding.go index 0264adf2b1..cb79d71978 100644 --- a/go/libraries/doltcore/sqle/index/z_encoding.go +++ b/go/libraries/doltcore/sqle/index/z_encoding.go @@ -21,6 +21,7 @@ import ( "math/bits" "sort" + "github.com/dolthub/dolt/go/store/val" "github.com/dolthub/go-mysql-server/sql/expression/function/spatial" "github.com/dolthub/go-mysql-server/sql/types" ) @@ -150,13 +151,29 @@ func ZSort(points []types.Point) []types.Point { return points } -// ZAddr converts the GeometryValue into a key: (min_z_val, level) +// ZCell converts the GeometryValue into a Cell // Note: there is an inefficiency here where small polygons may be placed into a level that's significantly larger -func ZAddr(v types.GeometryValue) [17]byte { +func ZCell(v types.GeometryValue) val.Cell { bbox := spatial.FindBBox(v) zMin := ZValue(types.Point{X: bbox[0], Y: bbox[1]}) zMax := ZValue(types.Point{X: bbox[2], Y: bbox[3]}) + cell := val.Cell{} + binary.BigEndian.PutUint64(cell.ZValue[:], zMin[0]) + binary.BigEndian.PutUint64(cell.ZValue[8:], zMin[1]) + if res := zMin[0] ^ zMax[0]; res != 0 { + cell.Level = byte(64 - bits.LeadingZeros64(res)/2) + } else { + cell.Level = byte(32 - bits.LeadingZeros64(zMin[1]^zMax[1])/2) + } + return cell +} + +// ZAddr converts the GeometryValue into a key: (level, min_z_val) +func ZAddr(v types.GeometryValue) [17]byte { + bbox := spatial.FindBBox(v) + zMin := ZValue(types.Point{X: bbox[0], Y: bbox[1]}) + zMax := ZValue(types.Point{X: bbox[2], Y: bbox[3]}) addr := [17]byte{} binary.BigEndian.PutUint64(addr[1:], zMin[0]) binary.BigEndian.PutUint64(addr[9:], zMin[1]) @@ -172,8 +189,11 @@ func ZAddr(v types.GeometryValue) [17]byte { // Note: there is an inefficiency here where small polygons may be placed into a level that's significantly larger func ZAddrSort(geoms []types.GeometryValue) []types.GeometryValue { sort.Slice(geoms, func(i, j int) bool { - zi, zj := ZAddr(geoms[i]), ZAddr(geoms[j]) - return bytes.Compare(zi[:], zj[:]) < 0 + zi, zj := ZCell(geoms[i]), ZCell(geoms[j]) + if zi.Level != zj.Level { + return zi.Level < zj.Level + } + return bytes.Compare(zi.ZValue[:], zj.ZValue[:]) < 0 }) return geoms } diff --git a/go/store/val/codec.go b/go/store/val/codec.go index e68fe21ace..f2d65a8d03 100644 --- a/go/store/val/codec.go +++ b/go/store/val/codec.go @@ -62,8 +62,8 @@ const ( bytesAddrEnc ByteSize = hash.ByteLen commitAddrEnc ByteSize = hash.ByteLen stringAddrEnc ByteSize = hash.ByteLen - jsonAddrEnc ByteSize = hash.ByteLen - zAddrSize ByteSize = 17 + jsonAddrEnc ByteSize = hash.ByteLen + cellSize ByteSize = 17 ) type Encoding byte @@ -92,8 +92,8 @@ const ( BytesAddrEnc = Encoding(serial.EncodingBytesAddr) CommitAddrEnc = Encoding(serial.EncodingCommitAddr) StringAddrEnc = Encoding(serial.EncodingStringAddr) - JSONAddrEnc = Encoding(serial.EncodingJSONAddr) - ZAddrEnc = Encoding(serial.EncodingZAddr) + JSONAddrEnc = Encoding(serial.EncodingJSONAddr) + CellEnc = Encoding(serial.EncodingCell) sentinel Encoding = 127 ) @@ -625,17 +625,30 @@ func stringFromBytes(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } -func compareZAddr(l, r [zAddrSize]byte) int { - return bytes.Compare(l[:], r[:]) +// Cell is a representation of a subregion for Spatial Indexes +// Level encodes the size of the region +// ZValue is the z-value encoding of the minimum point of the bbox of a geometry +type Cell struct { + Level byte + ZValue [16]byte } -func readZAddr(val []byte) (res [zAddrSize]byte) { - expectSize(val, zAddrSize) - copy(res[:zAddrSize], val[:zAddrSize]) +func compareCell(l, r Cell) int { + if l.Level != r.Level { + return int(l.Level - r.Level) + } + return bytes.Compare(l.ZValue[:], r.ZValue[:]) +} + +func readCell(val []byte) (res Cell) { + expectSize(val, cellSize) + res.Level = val[0] + copy(res.ZValue[:], val[1:]) return } -func writeZAddr(buf []byte, v [zAddrSize]byte) { - expectSize(buf, zAddrSize) - copy(buf, v[:zAddrSize]) +func writeCell(buf []byte, v Cell) { + expectSize(buf, cellSize) + buf[0] = v.Level + copy(buf[1:], v.ZValue[:]) } diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index 4f58ccc72c..cfa9523e10 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -243,8 +243,8 @@ func TestCompare(t *testing.T) { // z-address { typ: Type{Enc: StringEnc}, - l: encZAddr([zAddrSize]byte{}), - r: encZAddr([zAddrSize]byte{}), + l: encZAddr([cellSize]byte{}), + r: encZAddr([cellSize]byte{}), cmp: 0, }, } @@ -304,9 +304,9 @@ func encStr(s string) []byte { return buf } -func encZAddr(z [zAddrSize]byte) []byte { - buf := make([]byte, zAddrSize) - writeZAddr(buf, z) +func encZAddr(z [cellSize]byte) []byte { + buf := make([]byte, cellSize) + writeCell(buf, z) return buf } diff --git a/go/store/val/tuple_builder.go b/go/store/val/tuple_builder.go index 3c00c15095..b7daa34a1d 100644 --- a/go/store/val/tuple_builder.go +++ b/go/store/val/tuple_builder.go @@ -367,11 +367,11 @@ func (tb *TupleBuilder) ensureCapacity(sz ByteSize) { } } -// PutZAddr writes a Z-Address to the ith field of the Tuple being built. -func (tb *TupleBuilder) PutZAddr(i int, v [zAddrSize]byte) { - tb.Desc.expectEncoding(i, ZAddrEnc) - tb.ensureCapacity(zAddrSize) - tb.fields[i] = tb.buf[tb.pos : tb.pos+zAddrSize] - writeZAddr(tb.fields[i], v) - tb.pos += zAddrSize +// PutCell writes a Cell to the ith field of the Tuple being built. +func (tb *TupleBuilder) PutCell(i int, v Cell) { + tb.Desc.expectEncoding(i, CellEnc) + tb.ensureCapacity(cellSize) + tb.fields[i] = tb.buf[tb.pos : tb.pos+cellSize] + writeCell(tb.fields[i], v) + tb.pos += cellSize } diff --git a/go/store/val/tuple_compare.go b/go/store/val/tuple_compare.go index 6c524f7680..23fe9cf72e 100644 --- a/go/store/val/tuple_compare.go +++ b/go/store/val/tuple_compare.go @@ -143,8 +143,8 @@ func compare(typ Type, left, right []byte) int { return compareAddr(readAddr(left), readAddr(right)) case StringAddrEnc: return compareAddr(readAddr(left), readAddr(right)) - case ZAddrEnc: - return compareZAddr(readZAddr(left), readZAddr(right)) + case CellEnc: + return compareCell(readCell(left), readCell(right)) default: panic("unknown encoding") } diff --git a/go/store/val/tuple_descriptor.go b/go/store/val/tuple_descriptor.go index 9a2139ff15..aa7d934334 100644 --- a/go/store/val/tuple_descriptor.go +++ b/go/store/val/tuple_descriptor.go @@ -468,11 +468,11 @@ func (td TupleDesc) expectEncoding(i int, encodings ...Encoding) { panic("incorrect value encoding") } -func (td TupleDesc) GetZAddr(i int, tup Tuple) (v [zAddrSize]byte, ok bool) { - td.expectEncoding(i, ZAddrEnc) +func (td TupleDesc) GetCell(i int, tup Tuple) (v Cell, ok bool) { + td.expectEncoding(i, CellEnc) b := td.GetField(i, tup) if b != nil { - v = readZAddr(b) + v = readCell(b) ok = true } return @@ -572,6 +572,8 @@ func formatValue(enc Encoding, value []byte) string { return hex.EncodeToString(value) case CommitAddrEnc: return hex.EncodeToString(value) + case CellEnc: + return hex.EncodeToString(value) default: return string(value) } From 71f960b8ed5a595cc60681945656405df695d0d9 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 25 Jan 2023 14:00:07 -0800 Subject: [PATCH 12/15] small fixes --- go/gen/fb/serial/encoding.go | 4 ++-- go/libraries/doltcore/sqle/index/z_encoding.go | 7 ++----- go/serial/encoding.fbs | 2 +- go/store/val/codec_test.go | 8 ++++---- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/go/gen/fb/serial/encoding.go b/go/gen/fb/serial/encoding.go index 818937b90f..51feb06d16 100644 --- a/go/gen/fb/serial/encoding.go +++ b/go/gen/fb/serial/encoding.go @@ -78,7 +78,7 @@ var EnumNamesEncoding = map[Encoding]string{ EncodingCommitAddr: "CommitAddr", EncodingStringAddr: "StringAddr", EncodingJSONAddr: "JSONAddr", - EncodingCell: "ZAddr", + EncodingCell: "Cell", EncodingString: "String", EncodingBytes: "Bytes", EncodingDecimal: "Decimal", @@ -110,7 +110,7 @@ var EnumValuesEncoding = map[string]Encoding{ "CommitAddr": EncodingCommitAddr, "StringAddr": EncodingStringAddr, "JSONAddr": EncodingJSONAddr, - "ZAddr": EncodingCell, + "Cell": EncodingCell, "String": EncodingString, "Bytes": EncodingBytes, "Decimal": EncodingDecimal, diff --git a/go/libraries/doltcore/sqle/index/z_encoding.go b/go/libraries/doltcore/sqle/index/z_encoding.go index cb79d71978..43a237a7a8 100644 --- a/go/libraries/doltcore/sqle/index/z_encoding.go +++ b/go/libraries/doltcore/sqle/index/z_encoding.go @@ -189,11 +189,8 @@ func ZAddr(v types.GeometryValue) [17]byte { // Note: there is an inefficiency here where small polygons may be placed into a level that's significantly larger func ZAddrSort(geoms []types.GeometryValue) []types.GeometryValue { sort.Slice(geoms, func(i, j int) bool { - zi, zj := ZCell(geoms[i]), ZCell(geoms[j]) - if zi.Level != zj.Level { - return zi.Level < zj.Level - } - return bytes.Compare(zi.ZValue[:], zj.ZValue[:]) < 0 + zi, zj := ZAddr(geoms[i]), ZAddr(geoms[j]) + return bytes.Compare(zi[:], zj[:]) < 0 }) return geoms } diff --git a/go/serial/encoding.fbs b/go/serial/encoding.fbs index 7928413fe3..2c8b8f1109 100644 --- a/go/serial/encoding.fbs +++ b/go/serial/encoding.fbs @@ -39,7 +39,7 @@ enum Encoding : uint8 { CommitAddr = 22, StringAddr = 23, JSONAddr = 24, - ZAddr = 25, + Cell = 25, // variable width String = 128, diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index cfa9523e10..581d59832c 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -243,8 +243,8 @@ func TestCompare(t *testing.T) { // z-address { typ: Type{Enc: StringEnc}, - l: encZAddr([cellSize]byte{}), - r: encZAddr([cellSize]byte{}), + l: encCell(Cell{}), + r: encCell(Cell{}), cmp: 0, }, } @@ -304,9 +304,9 @@ func encStr(s string) []byte { return buf } -func encZAddr(z [cellSize]byte) []byte { +func encCell(c Cell) []byte { buf := make([]byte, cellSize) - writeCell(buf, z) + writeCell(buf, c) return buf } From 972b9f18231382d7142bbc083f90a52721829b88 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 25 Jan 2023 14:08:50 -0800 Subject: [PATCH 13/15] reformat --- go/libraries/doltcore/sqle/enginetest/dolt_queries.go | 2 +- go/libraries/doltcore/sqle/index/z_encoding.go | 3 ++- go/store/val/codec.go | 10 +++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index bae3f80f74..bb12cf45bf 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -2067,7 +2067,7 @@ var LogTableFunctionScriptTests = []queries.ScriptTest{ Expected: []sql.Row{{3}}, }, { - Query: "SELECT count(*) from dolt_log('main') join dolt_diff(@Commit1, @Commit2, 't') where commit_hash = to_commit;", + Query: "SELECT count(*) from dolt_log('main') join dolt_diff(@Commit1, @Commit2, 't') where commit_hash = to_commit;", Expected: []sql.Row{{2}}, }, }, diff --git a/go/libraries/doltcore/sqle/index/z_encoding.go b/go/libraries/doltcore/sqle/index/z_encoding.go index 43a237a7a8..5a5cd9d3b5 100644 --- a/go/libraries/doltcore/sqle/index/z_encoding.go +++ b/go/libraries/doltcore/sqle/index/z_encoding.go @@ -21,9 +21,10 @@ import ( "math/bits" "sort" - "github.com/dolthub/dolt/go/store/val" "github.com/dolthub/go-mysql-server/sql/expression/function/spatial" "github.com/dolthub/go-mysql-server/sql/types" + + "github.com/dolthub/dolt/go/store/val" ) // LexFloat maps the float64 into an uint64 representation in lexicographical order diff --git a/go/store/val/codec.go b/go/store/val/codec.go index f2d65a8d03..79594d38d8 100644 --- a/go/store/val/codec.go +++ b/go/store/val/codec.go @@ -62,8 +62,8 @@ const ( bytesAddrEnc ByteSize = hash.ByteLen commitAddrEnc ByteSize = hash.ByteLen stringAddrEnc ByteSize = hash.ByteLen - jsonAddrEnc ByteSize = hash.ByteLen - cellSize ByteSize = 17 + jsonAddrEnc ByteSize = hash.ByteLen + cellSize ByteSize = 17 ) type Encoding byte @@ -92,8 +92,8 @@ const ( BytesAddrEnc = Encoding(serial.EncodingBytesAddr) CommitAddrEnc = Encoding(serial.EncodingCommitAddr) StringAddrEnc = Encoding(serial.EncodingStringAddr) - JSONAddrEnc = Encoding(serial.EncodingJSONAddr) - CellEnc = Encoding(serial.EncodingCell) + JSONAddrEnc = Encoding(serial.EncodingJSONAddr) + CellEnc = Encoding(serial.EncodingCell) sentinel Encoding = 127 ) @@ -629,7 +629,7 @@ func stringFromBytes(b []byte) string { // Level encodes the size of the region // ZValue is the z-value encoding of the minimum point of the bbox of a geometry type Cell struct { - Level byte + Level byte ZValue [16]byte } From bde8d9920d40e57039b01f64cc3b70e229e44dd6 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 25 Jan 2023 14:19:31 -0800 Subject: [PATCH 14/15] no get for cells --- go/libraries/doltcore/sqle/index/prolly_fields.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/index/prolly_fields.go b/go/libraries/doltcore/sqle/index/prolly_fields.go index 11b05fba7a..553bdcbf44 100644 --- a/go/libraries/doltcore/sqle/index/prolly_fields.go +++ b/go/libraries/doltcore/sqle/index/prolly_fields.go @@ -118,8 +118,6 @@ func GetField(ctx context.Context, td val.TupleDesc, i int, tup val.Tuple, ns tr } case val.CommitAddrEnc: v, ok = td.GetCommitAddr(i, tup) - case val.CellEnc: - v, ok = td.GetCell(i, tup) default: panic("unknown val.encoding") } From 2286df2e9547c6069e904b3cb728d9727645bb60 Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Wed, 25 Jan 2023 22:40:01 +0000 Subject: [PATCH 15/15] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/enginetest/dolt_queries.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index bb12cf45bf..bae3f80f74 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -2067,7 +2067,7 @@ var LogTableFunctionScriptTests = []queries.ScriptTest{ Expected: []sql.Row{{3}}, }, { - Query: "SELECT count(*) from dolt_log('main') join dolt_diff(@Commit1, @Commit2, 't') where commit_hash = to_commit;", + Query: "SELECT count(*) from dolt_log('main') join dolt_diff(@Commit1, @Commit2, 't') where commit_hash = to_commit;", Expected: []sql.Row{{2}}, }, },