mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-04-30 03:26:47 -05:00
Code Issues Packages Projects Releases 159 Wiki Activity

Merge main

Browse Source
This commit is contained in:
Zach Musgrave
2022-06-08 10:30:20 -07:00
parent 123c9dce96 9f247c33d3
commit 72521b1465
75 changed files with 3169 additions and 2378 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/cmd/dolt/commands/init.go
+15
View File
@@ -140,6 +140,21 @@ func (cmd InitCmd) Exec(ctx context.Context, commandStr string, args []string, d
return 1
}
configuration := make(map[string]string)
if apr.Contains(usernameParamName) {
configuration[env.UserNameKey] = name
}
if apr.Contains(emailParamName) {
configuration[env.UserEmailKey] = email
}
if len(configuration) > 0 {
err = dEnv.Config.WriteableConfig().SetStrings(configuration)
if err != nil {
cli.PrintErrln(color.RedString("Failed to store initial configuration. %s", err.Error()))
return 1
}
}
cli.Println(color.CyanString("Successfully initialized dolt data repository."))
return 0
}
go/cmd/dolt/commands/init_test.go
+47 -24
View File
@@ -18,16 +18,20 @@ import (
"context"
"testing"
"github.com/stretchr/testify/require"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
)
type initTest struct {
Name string
Args []string
GlobalConfig map[string]string
ExpectSuccess bool
}
func TestInit(t *testing.T) {
tests := []struct {
Name string
Args []string
GlobalConfig map[string]string
ExpectSuccess bool
}{
tests := []initTest{
{
"Command Line name and email",
[]string{"-name", "Bill Billerson", "-email", "bigbillieb@fake.horse"},
@@ -65,18 +69,15 @@ func TestInit(t *testing.T) {
result := InitCmd{}.Exec(context.Background(), "dolt init", test.Args, dEnv)
if (result == 0) != test.ExpectSuccess {
t.Error(test.Name, "- Expected success:", test.ExpectSuccess, "result:", result == 0)
} else if test.ExpectSuccess {
// succceeded as expected
if !dEnv.HasDoltDir() {
t.Error(test.Name, "- .dolt dir should exist after initialization")
}
require.Equalf(t, test.ExpectSuccess, result == 0, "- Expected success: %t; result: %t;", test.ExpectSuccess, result == 0)
if test.ExpectSuccess {
require.True(t, dEnv.HasDoltDir(), "- .dolt dir should exist after initialization")
testLocalConfigValue(t, dEnv, test, usernameParamName, env.UserNameKey)
testLocalConfigValue(t, dEnv, test, emailParamName, env.UserEmailKey)
} else {
// failed as expected
if dEnv.HasDoltDir() {
t.Error(test.Name, "- dolt directory shouldn't exist after failure to initialize")
}
require.False(t, dEnv.HasDoltDir(),
"- dolt directory shouldn't exist after failure to initialize")
}
})
}
@@ -84,15 +85,37 @@ func TestInit(t *testing.T) {
func TestInitTwice(t *testing.T) {
dEnv := createUninitializedEnv()
result := InitCmd{}.Exec(context.Background(), "dolt init", []string{"-name", "Bill Billerson", "-email", "bigbillieb@fake.horse"}, dEnv)
result := InitCmd{}.Exec(context.Background(), "dolt init",
[]string{"-name", "Bill Billerson", "-email", "bigbillieb@fake.horse"}, dEnv)
require.True(t, result == 0, "First init should succeed")
if result != 0 {
t.Error("First init should succeed")
result = InitCmd{}.Exec(context.Background(), "dolt init",
[]string{"-name", "Bill Billerson", "-email", "bigbillieb@fake.horse"}, dEnv)
require.True(t, result != 0, "Second init should fail")
}
// testLocalConfigValue tests that local config data is set correctly when the specified argument
// is present in the command line args, and is not set when the argument is not present.
func testLocalConfigValue(t *testing.T, dEnv *env.DoltEnv, test initTest, argKey, envKey string) {
localConfig, ok := dEnv.Config.GetConfig(env.LocalConfig)
require.True(t, ok, "- Unable to load local configuration")
found := false
expectedValue := ""
for i := 0; i <= len(test.Args)-2; i = i + 2 {
if test.Args[i] == "-"+argKey {
found = true
expectedValue = test.Args[i+1]
}
}
result = InitCmd{}.Exec(context.Background(), "dolt init", []string{"-name", "Bill Billerson", "-email", "bigbillieb@fake.horse"}, dEnv)
if result == 0 {
t.Error("Second init should fail")
actualValue, err := localConfig.GetString(envKey)
if found {
require.NoErrorf(t, err, "- Expected '%s', but not found in local config; error: %v",
expectedValue, err)
require.Equalf(t, expectedValue, actualValue, "- Expected '%s' in local config, but found '%s'",
expectedValue, actualValue)
} else {
require.Errorf(t, err, "- Expected nothing in local config, but found '%s'", actualValue)
}
}
go/cmd/dolt/commands/roots.go
+7 -3
View File
@@ -34,6 +34,7 @@ import (
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/nbs"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/spec"
"github.com/dolthub/dolt/go/store/types"
)
@@ -61,11 +62,11 @@ func (cmd RootsCmd) RequiresRepo() bool {
// Description returns a description of the command
func (cmd RootsCmd) Description() string {
return "Displays the current Dolt cli version."
return "Displays store root values (or potential store root values) that we find in the current database."
}
func (cmd RootsCmd) GatedForNBF(nbf *types.NomsBinFormat) bool {
return types.IsFormat_DOLT_1(nbf)
return false
}
// CreateMarkdown creates a markdown file containing the helptext for the command at the given path
@@ -152,7 +153,10 @@ func (cmd RootsCmd) processTableFile(ctx context.Context, path string, modified
}
} else if sm, ok := value.(types.SerialMessage); ok {
if serial.GetFileID([]byte(sm)) == serial.StoreRootFileID {
err := types.WriteEncodedValue(ctx, cli.OutStream, value)
msg := serial.GetRootAsStoreRoot([]byte(sm), 0)
ambytes := msg.AddressMapBytes()
node := tree.NodeFromBytes(ambytes)
err := tree.OutputAddressMapNode(cli.OutStream, node)
if err != nil {
return false, err
}
go/cmd/dolt/dolt.go
+1 -1
View File
@@ -53,7 +53,7 @@ import (
)
const (
Version = "0.40.5"
Version = "0.40.6"
)
var dumpDocsCommand = &commands.DumpDocsCmd{}
go/gen/fb/serial/prolly.go
-128
View File
@@ -396,134 +396,6 @@ func ProllyTreeNodeEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
/// Refmap has been deprecated in favor of AddressMap
type RefMap struct {
_tab flatbuffers.Table
}
func GetRootAsRefMap(buf []byte, offset flatbuffers.UOffsetT) *RefMap {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &RefMap{}
x.Init(buf, n+offset)
return x
}
func GetSizePrefixedRootAsRefMap(buf []byte, offset flatbuffers.UOffsetT) *RefMap {
n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])
x := &RefMap{}
x.Init(buf, n+offset+flatbuffers.SizeUint32)
return x
}
func (rcv *RefMap) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *RefMap) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *RefMap) Names(j int) []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.ByteVector(a + flatbuffers.UOffsetT(j*4))
}
return nil
}
func (rcv *RefMap) NamesLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func (rcv *RefMap) RefArray(j int) byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1))
}
return 0
}
func (rcv *RefMap) RefArrayLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func (rcv *RefMap) RefArrayBytes() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
func (rcv *RefMap) MutateRefArray(j int, n byte) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.MutateByte(a+flatbuffers.UOffsetT(j*1), n)
}
return false
}
func (rcv *RefMap) TreeCount() uint64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.GetUint64(o + rcv._tab.Pos)
}
return 0
}
func (rcv *RefMap) MutateTreeCount(n uint64) bool {
return rcv._tab.MutateUint64Slot(8, n)
}
func (rcv *RefMap) TreeLevel() byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
if o != 0 {
return rcv._tab.GetByte(o + rcv._tab.Pos)
}
return 0
}
func (rcv *RefMap) MutateTreeLevel(n byte) bool {
return rcv._tab.MutateByteSlot(10, n)
}
func RefMapStart(builder *flatbuffers.Builder) {
builder.StartObject(4)
}
func RefMapAddNames(builder *flatbuffers.Builder, names flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(names), 0)
}
func RefMapStartNamesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func RefMapAddRefArray(builder *flatbuffers.Builder, refArray flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(refArray), 0)
}
func RefMapStartRefArrayVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(1, numElems, 1)
}
func RefMapAddTreeCount(builder *flatbuffers.Builder, treeCount uint64) {
builder.PrependUint64Slot(2, treeCount, 0)
}
func RefMapAddTreeLevel(builder *flatbuffers.Builder, treeLevel byte) {
builder.PrependByteSlot(3, treeLevel, 0)
}
func RefMapEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}
type CommitClosure struct {
_tab flatbuffers.Table
}
go/gen/fb/serial/rootvalue.go
+31 -7
View File
@@ -59,19 +59,40 @@ func (rcv *RootValue) MutateFeatureVersion(n int64) bool {
return rcv._tab.MutateInt64Slot(4, n)
}
func (rcv *RootValue) Tables(obj *RefMap) *RefMap {
func (rcv *RootValue) Tables(j int) byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(RefMap)
}
obj.Init(rcv._tab.Bytes, x)
return obj
a := rcv._tab.Vector(o)
return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1))
}
return 0
}
func (rcv *RootValue) TablesLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func (rcv *RootValue) TablesBytes() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
func (rcv *RootValue) MutateTables(j int, n byte) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.MutateByte(a+flatbuffers.UOffsetT(j*1), n)
}
return false
}
func (rcv *RootValue) ForeignKeyAddr(j int) byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
@@ -149,6 +170,9 @@ func RootValueAddFeatureVersion(builder *flatbuffers.Builder, featureVersion int
func RootValueAddTables(builder *flatbuffers.Builder, tables flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(tables), 0)
}
func RootValueStartTablesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(1, numElems, 1)
}
func RootValueAddForeignKeyAddr(builder *flatbuffers.Builder, foreignKeyAddr flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(foreignKeyAddr), 0)
}
go/gen/fb/serial/storeroot.go
+33 -9
View File
@@ -47,24 +47,48 @@ func (rcv *StoreRoot) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *StoreRoot) Refs(obj *RefMap) *RefMap {
func (rcv *StoreRoot) AddressMap(j int) byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(RefMap)
}
obj.Init(rcv._tab.Bytes, x)
return obj
a := rcv._tab.Vector(o)
return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1))
}
return 0
}
func (rcv *StoreRoot) AddressMapLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func (rcv *StoreRoot) AddressMapBytes() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
func (rcv *StoreRoot) MutateAddressMap(j int, n byte) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.MutateByte(a+flatbuffers.UOffsetT(j*1), n)
}
return false
}
func StoreRootStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func StoreRootAddRefs(builder *flatbuffers.Builder, refs flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(refs), 0)
func StoreRootAddAddressMap(builder *flatbuffers.Builder, addressMap flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(addressMap), 0)
}
func StoreRootStartAddressMapVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(1, numElems, 1)
}
func StoreRootEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
go/gen/fb/serial/table.go
+31 -7
View File
@@ -115,19 +115,40 @@ func (rcv *Table) MutatePrimaryIndex(j int, n byte) bool {
return false
}
func (rcv *Table) SecondaryIndexes(obj *RefMap) *RefMap {
func (rcv *Table) SecondaryIndexes(j int) byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(RefMap)
}
obj.Init(rcv._tab.Bytes, x)
return obj
a := rcv._tab.Vector(o)
return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1))
}
return 0
}
func (rcv *Table) SecondaryIndexesLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func (rcv *Table) SecondaryIndexesBytes() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
func (rcv *Table) MutateSecondaryIndexes(j int, n byte) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
a := rcv._tab.Vector(o)
return rcv._tab.MutateByte(a+flatbuffers.UOffsetT(j*1), n)
}
return false
}
func (rcv *Table) AutoIncrementValue() uint64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
if o != 0 {
@@ -205,6 +226,9 @@ func TableStartPrimaryIndexVector(builder *flatbuffers.Builder, numElems int) fl
func TableAddSecondaryIndexes(builder *flatbuffers.Builder, secondaryIndexes flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(secondaryIndexes), 0)
}
func TableStartSecondaryIndexesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(1, numElems, 1)
}
func TableAddAutoIncrementValue(builder *flatbuffers.Builder, autoIncrementValue uint64) {
builder.PrependUint64Slot(3, autoIncrementValue, 0)
}
go/go.mod
+145 -145
View File
@@ -1,145 +1,145 @@
module github.com/dolthub/dolt/go
require (
cloud.google.com/go/storage v1.12.0
github.com/BurntSushi/toml v0.3.1
github.com/HdrHistogram/hdrhistogram-go v1.0.0
github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible
github.com/aws/aws-sdk-go v1.32.6
github.com/bcicen/jstream v1.0.0
github.com/boltdb/bolt v1.3.1
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/denisbrodbeck/machineid v1.0.1
github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20201005193433-3ee972b1d078
github.com/dolthub/fslock v0.0.3
github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371
github.com/dolthub/mmap-go v1.0.4-0.20201107010347-f9f2a9588a66
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
github.com/dolthub/vitess v0.0.0-20220603212614-514e62ec66cd
github.com/dustin/go-humanize v1.0.0
github.com/fatih/color v1.9.0
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
github.com/go-openapi/errors v0.19.6 // indirect
github.com/go-openapi/strfmt v0.19.5 // indirect
github.com/go-sql-driver/mysql v1.6.0
github.com/gocraft/dbr/v2 v2.7.2
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.1
github.com/google/go-cmp v0.5.7
github.com/google/uuid v1.2.0
github.com/jedib0t/go-pretty v4.3.1-0.20191104025401-85fe5d6a7c4d+incompatible
github.com/jpillora/backoff v1.0.0
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d
github.com/mattn/go-isatty v0.0.12
github.com/mattn/go-runewidth v0.0.9
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b
github.com/mitchellh/mapstructure v1.3.2 // indirect
github.com/opentracing/opentracing-go v1.2.0
github.com/pkg/errors v0.9.1
github.com/pkg/profile v1.5.0
github.com/rivo/uniseg v0.1.0
github.com/sergi/go-diff v1.1.0 // indirect
github.com/shopspring/decimal v1.2.0
github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6
github.com/sirupsen/logrus v1.8.1
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
github.com/spf13/cobra v1.0.0
github.com/stretchr/testify v1.7.0
github.com/tealeg/xlsx v1.0.5
github.com/tklauser/go-sysconf v0.3.9 // indirect
github.com/uber/jaeger-client-go v2.25.0+incompatible
github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
go.mongodb.org/mongo-driver v1.7.0 // indirect
go.uber.org/zap v1.15.0
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20220111092808-5a964db01320
google.golang.org/api v0.32.0
google.golang.org/grpc v1.37.0
google.golang.org/protobuf v1.27.1
gopkg.in/square/go-jose.v2 v2.5.1
gopkg.in/src-d/go-errors.v1 v1.0.0
gopkg.in/yaml.v2 v2.3.0
)
require (
github.com/dolthub/go-mysql-server v0.11.1-0.20220608172308-2d1ad46ff2a0
github.com/google/flatbuffers v2.0.6+incompatible
github.com/gosuri/uilive v0.0.4
github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6
github.com/prometheus/client_golang v1.11.0
github.com/shirou/gopsutil/v3 v3.22.1
github.com/xitongsys/parquet-go v1.6.1
github.com/xitongsys/parquet-go-source v0.0.0-20211010230925-397910c5e371
github.com/zeebo/xxh3 v1.0.2
golang.org/x/text v0.3.7
gonum.org/v1/plot v0.11.0
)
require (
cloud.google.com/go v0.66.0 // indirect
git.sr.ht/~sbinet/gg v0.3.1 // indirect
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-fonts/liberation v0.2.0 // indirect
github.com/go-kit/kit v0.10.0 // indirect
github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-pdf/fpdf v0.6.0 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/googleapis/gax-go/v2 v2.0.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/klauspost/compress v1.10.10 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/lestrrat-go/strftime v1.0.4 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-colorable v0.1.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mitchellh/hashstructure v1.1.0 // indirect
github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 // indirect
github.com/pierrec/lz4/v4 v4.1.6 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/src-d/go-oniguruma v1.1.0 // indirect
github.com/tklauser/numcpus v0.3.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opencensus.io v0.22.4 // indirect
go.uber.org/atomic v1.6.0 // indirect
go.uber.org/multierr v1.5.0 // indirect
golang.org/x/image v0.0.0-20220302094943-723b81ca9867 // indirect
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
golang.org/x/tools v0.1.10 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210506142907-4a47615972c2 // indirect
gopkg.in/yaml.v3 v3.0.0 // indirect
)
replace (
github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi => ./gen/proto/dolt/services/eventsapi
github.com/oliveagle/jsonpath => github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474
)
go 1.18
module github.com/dolthub/dolt/go
require (
cloud.google.com/go/storage v1.12.0
github.com/BurntSushi/toml v0.3.1
github.com/HdrHistogram/hdrhistogram-go v1.0.0
github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible
github.com/aws/aws-sdk-go v1.32.6
github.com/bcicen/jstream v1.0.0
github.com/boltdb/bolt v1.3.1
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/denisbrodbeck/machineid v1.0.1
github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20201005193433-3ee972b1d078
github.com/dolthub/fslock v0.0.3
github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371
github.com/dolthub/mmap-go v1.0.4-0.20201107010347-f9f2a9588a66
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
github.com/dolthub/vitess v0.0.0-20220603212614-514e62ec66cd
github.com/dustin/go-humanize v1.0.0
github.com/fatih/color v1.9.0
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
github.com/go-openapi/errors v0.19.6 // indirect
github.com/go-openapi/strfmt v0.19.5 // indirect
github.com/go-sql-driver/mysql v1.6.0
github.com/gocraft/dbr/v2 v2.7.2
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.1
github.com/google/go-cmp v0.5.7
github.com/google/uuid v1.2.0
github.com/jedib0t/go-pretty v4.3.1-0.20191104025401-85fe5d6a7c4d+incompatible
github.com/jpillora/backoff v1.0.0
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d
github.com/mattn/go-isatty v0.0.12
github.com/mattn/go-runewidth v0.0.9
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b
github.com/mitchellh/mapstructure v1.3.2 // indirect
github.com/opentracing/opentracing-go v1.2.0
github.com/pkg/errors v0.9.1
github.com/pkg/profile v1.5.0
github.com/rivo/uniseg v0.1.0
github.com/sergi/go-diff v1.1.0 // indirect
github.com/shopspring/decimal v1.2.0
github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6
github.com/sirupsen/logrus v1.8.1
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
github.com/spf13/cobra v1.0.0
github.com/stretchr/testify v1.7.0
github.com/tealeg/xlsx v1.0.5
github.com/tklauser/go-sysconf v0.3.9 // indirect
github.com/uber/jaeger-client-go v2.25.0+incompatible
github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
go.mongodb.org/mongo-driver v1.7.0 // indirect
go.uber.org/zap v1.15.0
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20220111092808-5a964db01320
google.golang.org/api v0.32.0
google.golang.org/grpc v1.37.0
google.golang.org/protobuf v1.27.1
gopkg.in/square/go-jose.v2 v2.5.1
gopkg.in/src-d/go-errors.v1 v1.0.0
gopkg.in/yaml.v2 v2.3.0
)
require (
github.com/dolthub/go-mysql-server v0.11.1-0.20220608172308-2d1ad46ff2a0
github.com/google/flatbuffers v2.0.6+incompatible
github.com/gosuri/uilive v0.0.4
github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6
github.com/prometheus/client_golang v1.11.0
github.com/shirou/gopsutil/v3 v3.22.1
github.com/xitongsys/parquet-go v1.6.1
github.com/xitongsys/parquet-go-source v0.0.0-20211010230925-397910c5e371
github.com/zeebo/xxh3 v1.0.2
golang.org/x/text v0.3.7
gonum.org/v1/plot v0.11.0
)
require (
cloud.google.com/go v0.66.0 // indirect
git.sr.ht/~sbinet/gg v0.3.1 // indirect
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-fonts/liberation v0.2.0 // indirect
github.com/go-kit/kit v0.10.0 // indirect
github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-pdf/fpdf v0.6.0 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/googleapis/gax-go/v2 v2.0.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/klauspost/compress v1.10.10 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/lestrrat-go/strftime v1.0.4 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-colorable v0.1.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mitchellh/hashstructure v1.1.0 // indirect
github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 // indirect
github.com/pierrec/lz4/v4 v4.1.6 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/src-d/go-oniguruma v1.1.0 // indirect
github.com/tklauser/numcpus v0.3.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opencensus.io v0.22.4 // indirect
go.uber.org/atomic v1.6.0 // indirect
go.uber.org/multierr v1.5.0 // indirect
golang.org/x/image v0.0.0-20220302094943-723b81ca9867 // indirect
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
golang.org/x/tools v0.1.10 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210506142907-4a47615972c2 // indirect
gopkg.in/yaml.v3 v3.0.0 // indirect
)
replace (
github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi => ./gen/proto/dolt/services/eventsapi
github.com/oliveagle/jsonpath => github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474
)
go 1.18
go/go.sum
+1294 -1294
View File
File diff suppressed because it is too large Load Diff
go/libraries/doltcore/doltdb/durable/conflict_index.go
+7 -6
View File
@@ -20,6 +20,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -37,7 +38,7 @@ func RefFromConflictIndex(ctx context.Context, vrw types.ValueReadWriter, idx Co
return refFromNomsValue(ctx, vrw, idx.(nomsConflictIndex).index)
case types.Format_DOLT_1:
b := prolly.ValueFromConflictMap(idx.(prollyConflictIndex).index)
b := shim.ValueFromConflictMap(idx.(prollyConflictIndex).index)
return refFromNomsValue(ctx, vrw, b)
default:
@@ -56,10 +57,10 @@ func NewEmptyConflictIndex(ctx context.Context, vrw types.ValueReadWriter, oursS
return ConflictIndexFromNomsMap(m, vrw), nil
case types.Format_DOLT_1:
kd, oursVD := prolly.MapDescriptorsFromSchema(oursSch)
theirsVD := prolly.ValueDescriptorFromSchema(theirsSch)
baseVD := prolly.ValueDescriptorFromSchema(baseSch)
ns := tree.NewNodeStore(prolly.ChunkStoreFromVRW(vrw))
kd, oursVD := shim.MapDescriptorsFromSchema(oursSch)
theirsVD := shim.ValueDescriptorFromSchema(theirsSch)
baseVD := shim.ValueDescriptorFromSchema(baseSch)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
m := prolly.NewEmptyConflictMap(ns, kd, oursVD, theirsVD, baseVD)
@@ -106,7 +107,7 @@ func conflictIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ourSc
return ConflictIndexFromNomsMap(v.(types.Map), vrw), nil
case types.Format_DOLT_1:
m := prolly.ConflictMapFromValue(v, ourSch, theirSch, baseSch, vrw)
m := shim.ConflictMapFromValue(v, ourSch, theirSch, baseSch, vrw)
return ConflictIndexFromProllyMap(m), nil
default:
go/libraries/doltcore/doltdb/durable/index.go
+76 -36
View File
@@ -20,13 +20,10 @@ import (
"io"
"strings"
flatbuffers "github.com/google/flatbuffers/go"
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
@@ -84,7 +81,7 @@ func RefFromIndex(ctx context.Context, vrw types.ValueReadWriter, idx Index) (ty
return refFromNomsValue(ctx, vrw, idx.(nomsIndex).index)
case types.Format_DOLT_1:
b := prolly.ValueFromMap(idx.(prollyIndex).index)
b := shim.ValueFromMap(idx.(prollyIndex).index)
return refFromNomsValue(ctx, vrw, b)
default:
@@ -108,7 +105,7 @@ func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, sch schema.Sc
return IndexFromNomsMap(v.(types.Map), vrw), nil
case types.Format_DOLT_1:
pm := prolly.MapFromValue(v, sch, vrw)
pm := shim.MapFromValue(v, sch, vrw)
return IndexFromProllyMap(pm), nil
default:
@@ -127,8 +124,8 @@ func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, sch schema.Sc
return IndexFromNomsMap(m, vrw), nil
case types.Format_DOLT_1:
kd, vd := prolly.MapDescriptorsFromSchema(sch)
ns := tree.NewNodeStore(prolly.ChunkStoreFromVRW(vrw))
kd, vd := shim.MapDescriptorsFromSchema(sch)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
m, err := prolly.NewMapFromTuples(ctx, ns, kd, vd)
if err != nil {
return nil, err
@@ -147,6 +144,24 @@ type nomsIndex struct {
var _ Index = nomsIndex{}
func IterAllIndexes(
ctx context.Context,
sch schema.Schema,
set IndexSet,
cb func(name string, idx Index) error,
) error {
for _, def := range sch.Indexes().AllIndexes() {
idx, err := set.GetIndex(ctx, sch, def.Name())
if err != nil {
return err
}
if err = cb(def.Name(), idx); err != nil {
return err
}
}
return nil
}
// NomsMapFromIndex unwraps the Index and returns the underlying types.Map.
func NomsMapFromIndex(i Index) types.Map {
return i.(nomsIndex).index
@@ -234,7 +249,7 @@ func (i prollyIndex) Format() *types.NomsBinFormat {
// bytes implements Index.
func (i prollyIndex) bytes() ([]byte, error) {
return []byte(prolly.ValueFromMap(i.index).(types.TupleRowStorage)), nil
return []byte(shim.ValueFromMap(i.index).(types.TupleRowStorage)), nil
}
var _ Index = prollyIndex{}
@@ -305,10 +320,9 @@ func (i prollyIndex) AddColumnToRows(ctx context.Context, newCol string, newSche
// NewIndexSet returns an empty IndexSet.
func NewIndexSet(ctx context.Context, vrw types.ValueReadWriter) IndexSet {
if vrw.Format().UsesFlatbuffers() {
builder := flatbuffers.NewBuilder(24)
serial.RefMapStart(builder)
builder.Finish(serial.RefMapEnd(builder))
return doltDevIndexSet{vrw, serial.GetRootAsRefMap(builder.FinishedBytes(), 0)}
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
emptyam := prolly.NewEmptyAddressMap(ns)
return doltDevIndexSet{vrw, emptyam}
}
empty, _ := types.NewMap(ctx, vrw)
@@ -417,17 +431,20 @@ func mapFromIndexSet(ic IndexSet) types.Map {
type doltDevIndexSet struct {
vrw types.ValueReadWriter
msg *serial.RefMap
am prolly.AddressMap
}
var _ IndexSet = doltDevIndexSet{}
func (is doltDevIndexSet) HashOf() (hash.Hash, error) {
return types.SerialMessage(is.msg.Table().Bytes).Hash(is.vrw.Format())
return is.am.HashOf(), nil
}
func (is doltDevIndexSet) GetIndex(ctx context.Context, sch schema.Schema, name string) (Index, error) {
addr := datas.RefMapLookup(is.msg, name)
addr, err := is.am.Get(ctx, name)
if err != nil {
return nil, err
}
if addr.IsEmpty() {
return nil, fmt.Errorf("index %s not found in IndexSet", name)
}
@@ -444,12 +461,17 @@ func (is doltDevIndexSet) PutIndex(ctx context.Context, name string, idx Index)
return nil, err
}
builder := flatbuffers.NewBuilder(1024)
off := datas.RefMapApplyEdits(is.msg, builder, []datas.RefMapEdit{{Name: name, Addr: ref.TargetHash()}})
builder.Finish(off)
msg := serial.GetRootAsRefMap(builder.FinishedBytes(), 0)
ae := is.am.Editor()
err = ae.Update(ctx, name, ref.TargetHash())
if err != nil {
return nil, err
}
am, err := ae.Flush(ctx)
if err != nil {
return nil, err
}
return doltDevIndexSet{is.vrw, msg}, nil
return doltDevIndexSet{is.vrw, am}, nil
}
func (is doltDevIndexSet) PutNomsIndex(ctx context.Context, name string, idx types.Map) (IndexSet, error) {
@@ -457,30 +479,48 @@ func (is doltDevIndexSet) PutNomsIndex(ctx context.Context, name string, idx typ
}
func (is doltDevIndexSet) DropIndex(ctx context.Context, name string) (IndexSet, error) {
builder := flatbuffers.NewBuilder(1024)
off := datas.RefMapApplyEdits(is.msg, builder, []datas.RefMapEdit{{Name: name, Addr: hash.Hash{}}})
builder.Finish(off)
msg := serial.GetRootAsRefMap(builder.FinishedBytes(), 0)
return doltDevIndexSet{is.vrw, msg}, nil
ae := is.am.Editor()
err := ae.Delete(ctx, name)
if err != nil {
return nil, err
}
am, err := ae.Flush(ctx)
if err != nil {
return nil, err
}
return doltDevIndexSet{is.vrw, am}, nil
}
func (is doltDevIndexSet) RenameIndex(ctx context.Context, oldName, newName string) (IndexSet, error) {
addr := datas.RefMapLookup(is.msg, oldName)
addr, err := is.am.Get(ctx, oldName)
if err != nil {
return nil, err
}
if addr.IsEmpty() {
return nil, fmt.Errorf("index %s not found in IndexSet", oldName)
}
newaddr := datas.RefMapLookup(is.msg, newName)
newaddr, err := is.am.Get(ctx, newName)
if err != nil {
return nil, err
}
if !newaddr.IsEmpty() {
return nil, fmt.Errorf("index %s found in IndexSet when attempting to rename index", newName)
}
builder := flatbuffers.NewBuilder(1024)
off := datas.RefMapApplyEdits(is.msg, builder, []datas.RefMapEdit{
{Name: newName, Addr: addr},
{Name: oldName},
})
builder.Finish(off)
msg := serial.GetRootAsRefMap(builder.FinishedBytes(), 0)
ae := is.am.Editor()
err = ae.Update(ctx, newName, addr)
if err != nil {
return nil, err
}
err = ae.Delete(ctx, oldName)
if err != nil {
return nil, err
}
return doltDevIndexSet{is.vrw, msg}, nil
am, err := ae.Flush(ctx)
if err != nil {
return nil, err
}
return doltDevIndexSet{is.vrw, am}, nil
}
go/libraries/doltcore/doltdb/durable/table.go
+18 -9
View File
@@ -27,10 +27,10 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/encoding"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/pool"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -663,7 +663,7 @@ var _ Table = doltDevTable{}
type serialTableFields struct {
schema []byte
rows []byte
indexes *serial.RefMap
indexes prolly.AddressMap
conflictsdata []byte
conflictsours []byte
conflictstheirs []byte
@@ -676,9 +676,12 @@ func (fields serialTableFields) write() *serial.Table {
// TODO: Chance for a pool.
builder := flatbuffers.NewBuilder(1024)
indexesam := fields.indexes
indexesbytes := []byte(tree.ValueFromNode(indexesam.Node()).(types.TupleRowStorage))
schemaoff := builder.CreateByteVector(fields.schema)
rowsoff := builder.CreateByteVector(fields.rows)
indexesoff := datas.RefMapApplyEdits(fields.indexes, builder, nil)
indexesoff := builder.CreateByteVector(indexesbytes)
conflictsdataoff := builder.CreateByteVector(fields.conflictsdata)
conflictsoursoff := builder.CreateByteVector(fields.conflictsours)
conflictstheirsoff := builder.CreateByteVector(fields.conflictstheirs)
@@ -733,7 +736,7 @@ func newDoltDevTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.
msg := serialTableFields{
schema: schemaAddr[:],
rows: rowsbytes,
indexes: indexes.(doltDevIndexSet).msg,
indexes: indexes.(doltDevIndexSet).am,
conflictsdata: emptyhash[:],
conflictsours: emptyhash[:],
conflictstheirs: emptyhash[:],
@@ -797,7 +800,7 @@ func (t doltDevTable) GetTableRows(ctx context.Context) (Index, error) {
if err != nil {
return nil, err
}
m := prolly.MapFromValue(types.TupleRowStorage(rowbytes), sch, t.vrw)
m := shim.MapFromValue(types.TupleRowStorage(rowbytes), sch, t.vrw)
return IndexFromProllyMap(m), nil
}
}
@@ -816,13 +819,15 @@ func (t doltDevTable) SetTableRows(ctx context.Context, rows Index) (Table, erro
}
func (t doltDevTable) GetIndexes(ctx context.Context) (IndexSet, error) {
is := t.msg.SecondaryIndexes(nil)
return doltDevIndexSet{t.vrw, is}, nil
ambytes := t.msg.SecondaryIndexesBytes()
node := tree.NodeFromBytes(ambytes)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(t.vrw))
return doltDevIndexSet{t.vrw, prolly.NewAddressMap(node, ns)}, nil
}
func (t doltDevTable) SetIndexes(ctx context.Context, indexes IndexSet) (Table, error) {
fields := t.fields()
fields.indexes = indexes.(doltDevIndexSet).msg
fields.indexes = indexes.(doltDevIndexSet).am
msg := fields.write()
return doltDevTable{t.vrw, msg}, nil
}
@@ -983,11 +988,15 @@ func (t doltDevTable) clone() *serial.Table {
}
func (t doltDevTable) fields() serialTableFields {
ambytes := t.msg.SecondaryIndexesBytes()
node := tree.NodeFromBytes(ambytes)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(t.vrw))
conflicts := t.msg.Conflicts(nil)
return serialTableFields{
schema: t.msg.SchemaBytes(),
rows: t.msg.PrimaryIndexBytes(),
indexes: t.msg.SecondaryIndexes(nil),
indexes: prolly.NewAddressMap(node, ns),
conflictsdata: conflicts.DataBytes(),
conflictsours: conflicts.OurSchemaBytes(),
conflictstheirs: conflicts.TheirSchemaBytes(),
go/libraries/doltcore/doltdb/root_val.go
+63 -31
View File
@@ -28,8 +28,10 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/encoding"
"github.com/dolthub/dolt/go/libraries/utils/set"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -78,7 +80,7 @@ func tmIterAll(ctx context.Context, tm tableMap, cb func(name string, addr hash.
type rvStorage interface {
GetFeatureVersion() (FeatureVersion, bool, error)
GetTablesMap(ctx context.Context, vr types.ValueReader) (tableMap, error)
GetTablesMap(ctx context.Context, vr types.ValueReadWriter) (tableMap, error)
GetSuperSchemaMap(ctx context.Context, vr types.ValueReader) (types.Map, bool, error)
GetForeignKeyMap(ctx context.Context, vr types.ValueReader) (types.Map, bool, error)
@@ -108,7 +110,7 @@ func (r nomsRvStorage) GetFeatureVersion() (FeatureVersion, bool, error) {
}
}
func (r nomsRvStorage) GetTablesMap(context.Context, types.ValueReader) (tableMap, error) {
func (r nomsRvStorage) GetTablesMap(context.Context, types.ValueReadWriter) (tableMap, error) {
v, found, err := r.valueSt.MaybeGet(tablesKey)
if err != nil {
return nil, err
@@ -293,9 +295,13 @@ func isRootValue(nbf *types.NomsBinFormat, val types.Value) bool {
func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter) (*RootValue, error) {
if vrw.Format().UsesFlatbuffers() {
builder := flatbuffers.NewBuilder(80)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
emptyam := prolly.NewEmptyAddressMap(ns)
ambytes := []byte(tree.ValueFromNode(emptyam.Node()).(types.TupleRowStorage))
tablesoff := builder.CreateByteVector(ambytes)
var empty hash.Hash
serial.RefMapStart(builder)
tablesoff := serial.RefMapEnd(builder)
fkoff := builder.CreateByteVector(empty[:])
ssoff := builder.CreateByteVector(empty[:])
serial.RootValueStart(builder)
@@ -1365,33 +1371,36 @@ func (r fbRvStorage) GetFeatureVersion() (FeatureVersion, bool, error) {
return FeatureVersion(r.srv.FeatureVersion()), true, nil
}
func (r fbRvStorage) GetTablesMap(ctx context.Context, vr types.ValueReader) (tableMap, error) {
return fbTableMap{r.srv.Tables(nil)}, nil
func (r fbRvStorage) getAddressMap(vrw types.ValueReadWriter) prolly.AddressMap {
tbytes := r.srv.TablesBytes()
node := shim.NodeFromValue(types.TupleRowStorage(tbytes))
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
return prolly.NewAddressMap(node, ns)
}
func (r fbRvStorage) GetTablesMap(ctx context.Context, vrw types.ValueReadWriter) (tableMap, error) {
am := r.getAddressMap(vrw)
return fbTableMap{am}, nil
}
type fbTableMap struct {
*serial.RefMap
prolly.AddressMap
}
func (m fbTableMap) Get(ctx context.Context, name string) (hash.Hash, error) {
return datas.RefMapLookup(m.RefMap, name), nil
return m.AddressMap.Get(ctx, name)
}
func (m fbTableMap) Iter(ctx context.Context, cb func(string, hash.Hash) (bool, error)) error {
refs := m.RefArrayBytes()
for i := 0; i < m.NamesLength(); i++ {
off := i * 20
addr := hash.New(refs[off : off+20])
name := string(m.Names(i))
stop, err := cb(name, addr)
if err != nil {
var stop bool
return m.AddressMap.IterAll(ctx, func(n string, a hash.Hash) error {
if !stop {
var err error
stop, err = cb(n, a)
return err
}
if stop {
return nil
}
}
return nil
return nil
})
}
func (r fbRvStorage) GetSuperSchemaMap(ctx context.Context, vr types.ValueReader) (types.Map, bool, error) {
@@ -1435,30 +1444,53 @@ func (r fbRvStorage) SetSuperSchemaMap(ctx context.Context, vrw types.ValueReadW
func (r fbRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWriter, edits []tableEdit) (rvStorage, error) {
builder := flatbuffers.NewBuilder(80)
tables := r.srv.Tables(nil)
var rmedits []datas.RefMapEdit
am := r.getAddressMap(vrw)
ae := am.Editor()
for _, e := range edits {
if e.old_name != "" {
oldaddr := datas.RefMapLookup(tables, e.old_name)
newaddr := datas.RefMapLookup(tables, e.name)
oldaddr, err := am.Get(ctx, e.old_name)
if err != nil {
return nil, err
}
newaddr, err := am.Get(ctx, e.name)
if err != nil {
return nil, err
}
if oldaddr.IsEmpty() {
return nil, ErrTableNotFound
}
if !newaddr.IsEmpty() {
return nil, ErrTableExists
}
rmedits = append(rmedits, datas.RefMapEdit{Name: e.old_name})
rmedits = append(rmedits, datas.RefMapEdit{Name: e.name, Addr: oldaddr})
err = ae.Delete(ctx, e.old_name)
if err != nil {
return nil, err
}
err = ae.Update(ctx, e.name, oldaddr)
if err != nil {
return nil, err
}
} else {
if e.ref == nil {
rmedits = append(rmedits, datas.RefMapEdit{Name: e.name})
err := ae.Delete(ctx, e.name)
if err != nil {
return nil, err
}
} else {
rmedits = append(rmedits, datas.RefMapEdit{Name: e.name, Addr: e.ref.TargetHash()})
err := ae.Update(ctx, e.name, e.ref.TargetHash())
if err != nil {
return nil, err
}
}
}
}
tablesoff := datas.RefMapApplyEdits(tables, builder, rmedits)
am, err := ae.Flush(ctx)
if err != nil {
return nil, err
}
ambytes := []byte(tree.ValueFromNode(am.Node()).(types.TupleRowStorage))
tablesoff := builder.CreateByteVector(ambytes)
fkoff := builder.CreateByteVector(r.srv.ForeignKeyAddrBytes())
ssoff := builder.CreateByteVector(r.srv.SuperSchemasAddrBytes())
go/libraries/doltcore/env/actions/errors.go
Vendored
+2 -2
View File
@@ -25,8 +25,8 @@ type tblErrorType string
const (
tblErrInvalid tblErrorType = "invalid"
tblErrTypeNotExist tblErrorType = "do not exist"
tblErrTypeInConflict tblErrorType = "in conflict"
tblErrTypeConstViols tblErrorType = "has constraint violations"
tblErrTypeInConflict tblErrorType = "are in conflict"
tblErrTypeConstViols tblErrorType = "have constraint violations"
)
type TblError struct {
go/libraries/doltcore/merge/merge_prolly.go
+2 -1
View File
@@ -24,6 +24,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/pool"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
@@ -233,7 +234,7 @@ func newValueMerger(merged, leftSch, rightSch, baseSch schema.Schema, syncPool p
return &valueMerger{
numCols: n,
vD: prolly.ValueDescriptorFromSchema(merged),
vD: shim.ValueDescriptorFromSchema(merged),
leftMapping: leftMapping,
rightMapping: rightMapping,
baseMapping: baseMapping,
go/libraries/doltcore/merge/merge_test.go
+4 -3
View File
@@ -35,6 +35,7 @@ import (
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/pool"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
@@ -69,7 +70,7 @@ type rowV struct {
col1, col2 int
}
var vD = prolly.ValueDescriptorFromSchema(sch)
var vD = shim.ValueDescriptorFromSchema(sch)
var vB = val.NewTupleBuilder(vD)
var syncPool = pool.NewBuffPool()
@@ -420,7 +421,7 @@ func setupMergeTest(t *testing.T) (types.ValueReadWriter, *doltdb.RootValue, *do
vrw := ddb.ValueReadWriter()
sortTests(testRows)
ns := tree.NewNodeStore(prolly.ChunkStoreFromVRW(vrw))
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
var initialKVs []val.Tuple
var expectedKVs []val.Tuple
@@ -747,7 +748,7 @@ func buildLeftRightAncCommitsAndBranches(t *testing.T, ddb *doltdb.DoltDB, rootT
return root, mergeRoot, ancRoot
}
var kD = prolly.KeyDescriptorFromSchema(sch)
var kD = shim.KeyDescriptorFromSchema(sch)
var kB = val.NewTupleBuilder(kD)
func key(i int) val.Tuple {
go/libraries/doltcore/merge/row_merge_test.go
+3 -3
View File
@@ -22,7 +22,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
@@ -223,7 +223,7 @@ func TestRowMerge(t *testing.T) {
merged, isConflict := v.tryMerge(test.row, test.mergeRow, test.ancRow)
assert.Equal(t, test.expectConflict, isConflict)
vD := prolly.ValueDescriptorFromSchema(test.mergedSch)
vD := shim.ValueDescriptorFromSchema(test.mergedSch)
assert.Equal(t, vD.Format(test.expectedResult), vD.Format(merged))
})
}
@@ -342,7 +342,7 @@ func buildTup(sch schema.Schema, r []*int) val.Tuple {
return nil
}
vD := prolly.ValueDescriptorFromSchema(sch)
vD := shim.ValueDescriptorFromSchema(sch)
vB := val.NewTupleBuilder(vD)
for i, v := range r {
if v != nil {
go/libraries/doltcore/sqle/dfunctions/dolt_checkout.go
+2
View File
@@ -78,6 +78,8 @@ func DoDoltCheckout(ctx *sql.Context, args []string) (int, error) {
if newBranch, newBranchOk := apr.GetValue(cli.CheckoutCoBranch); newBranchOk {
if len(newBranch) == 0 {
err = errors.New("error: cannot checkout empty string")
} else if len(apr.Args) > 0 {
err = checkoutNewBranch(ctx, dbName, dbData, roots, newBranch, apr.Arg(0))
} else {
err = checkoutNewBranch(ctx, dbName, dbData, roots, newBranch, "")
}
go/libraries/doltcore/sqle/dfunctions/dolt_merge.go
+46 -37
View File
@@ -47,8 +47,13 @@ type DoltMergeFunc struct {
const DoltMergeWarningCode int = 1105 // Since this our own custom warning we'll use 1105, the code for an unknown error
const (
hasConflictsOrViolations int = 0
noConflictsOrViolations int = 1
noConflictsOrViolations int = 0
hasConflictsOrViolations int = 1
)
const (
threeWayMerge = 0
fastForwardMerge = 1
)
func (d DoltMergeFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
@@ -56,92 +61,96 @@ func (d DoltMergeFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, error)
if err != nil {
return noConflictsOrViolations, err
}
return DoDoltMerge(ctx, args)
hasConflicts, _, err := DoDoltMerge(ctx, args)
return hasConflicts, err
}
func DoDoltMerge(ctx *sql.Context, args []string) (int, error) {
// DoDoltMerge returns has_conflicts and fast_forward status
func DoDoltMerge(ctx *sql.Context, args []string) (int, int, error) {
dbName := ctx.GetCurrentDatabase()
if len(dbName) == 0 {
return noConflictsOrViolations, fmt.Errorf("Empty database name.")
return noConflictsOrViolations, threeWayMerge, fmt.Errorf("Empty database name.")
}
sess := dsess.DSessFromSess(ctx.Session)
apr, err := cli.CreateMergeArgParser().Parse(args)
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
if apr.ContainsAll(cli.SquashParam, cli.NoFFParam) {
return noConflictsOrViolations, fmt.Errorf("error: Flags '--%s' and '--%s' cannot be used together.\n", cli.SquashParam, cli.NoFFParam)
return noConflictsOrViolations, threeWayMerge, fmt.Errorf("error: Flags '--%s' and '--%s' cannot be used together.\n", cli.SquashParam, cli.NoFFParam)
}
ws, err := sess.WorkingSet(ctx, dbName)
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
roots, ok := sess.GetRoots(ctx, dbName)
if !ok {
return noConflictsOrViolations, sql.ErrDatabaseNotFound.New(dbName)
return noConflictsOrViolations, threeWayMerge, sql.ErrDatabaseNotFound.New(dbName)
}
if apr.Contains(cli.AbortParam) {
if !ws.MergeActive() {
return noConflictsOrViolations, fmt.Errorf("fatal: There is no merge to abort")
return noConflictsOrViolations, threeWayMerge, fmt.Errorf("fatal: There is no merge to abort")
}
ws, err = abortMerge(ctx, ws, roots)
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
err := sess.SetWorkingSet(ctx, dbName, ws)
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
err = sess.CommitWorkingSet(ctx, dbName, sess.GetTransaction())
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
return noConflictsOrViolations, nil
return noConflictsOrViolations, threeWayMerge, nil
}
branchName := apr.Arg(0)
mergeSpec, err := createMergeSpec(ctx, sess, dbName, apr, branchName)
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
ws, conflicts, err := mergeIntoWorkingSet(ctx, sess, roots, ws, dbName, mergeSpec)
ws, conflicts, fastForward, err := mergeIntoWorkingSet(ctx, sess, roots, ws, dbName, mergeSpec)
if err != nil {
return conflicts, err
return conflicts, fastForward, err
}
return conflicts, nil
return conflicts, fastForward, nil
}
// mergeIntoWorkingSet encapsulates server merge logic, switching between fast-forward, no fast-forward, merge commit,
// and merging into working set. Returns a new WorkingSet and whether there were merge conflicts. This currently
// persists merge commits in the database, but expects the caller to update the working set.
// mergeIntoWorkingSet encapsulates server merge logic, switching between
// fast-forward, no fast-forward, merge commit, and merging into working set.
// Returns a new WorkingSet, whether there were merge conflicts, and whether a
// fast-forward was performed. This currently persists merge commits in the
// database, but expects the caller to update the working set.
// TODO FF merging commit with constraint violations requires `constraint verify`
func mergeIntoWorkingSet(ctx *sql.Context, sess *dsess.DoltSession, roots doltdb.Roots, ws *doltdb.WorkingSet, dbName string, spec *merge.MergeSpec) (*doltdb.WorkingSet, int, error) {
func mergeIntoWorkingSet(ctx *sql.Context, sess *dsess.DoltSession, roots doltdb.Roots, ws *doltdb.WorkingSet, dbName string, spec *merge.MergeSpec) (*doltdb.WorkingSet, int, int, error) {
// todo: allow merges even when an existing merge is uncommitted
if ws.MergeActive() {
return ws, noConflictsOrViolations, doltdb.ErrMergeActive
return ws, noConflictsOrViolations, threeWayMerge, doltdb.ErrMergeActive
}
err := checkForUncommittedChanges(ctx, roots.Working, roots.Head)
if err != nil {
return ws, noConflictsOrViolations, err
return ws, noConflictsOrViolations, threeWayMerge, err
}
dbData, ok := sess.GetDbData(ctx, dbName)
if !ok {
return ws, noConflictsOrViolations, fmt.Errorf("failed to get dbData")
return ws, noConflictsOrViolations, threeWayMerge, fmt.Errorf("failed to get dbData")
}
canFF, err := spec.HeadC.CanFastForwardTo(ctx, spec.MergeC)
@@ -150,7 +159,7 @@ func mergeIntoWorkingSet(ctx *sql.Context, sess *dsess.DoltSession, roots doltdb
case doltdb.ErrIsAhead, doltdb.ErrUpToDate:
ctx.Warn(DoltMergeWarningCode, err.Error())
default:
return ws, noConflictsOrViolations, err
return ws, noConflictsOrViolations, threeWayMerge, err
}
}
@@ -162,25 +171,25 @@ func mergeIntoWorkingSet(ctx *sql.Context, sess *dsess.DoltSession, roots doltdb
// error message
wsErr := sess.SetWorkingSet(ctx, dbName, ws)
if wsErr != nil {
return ws, hasConflictsOrViolations, wsErr
return ws, hasConflictsOrViolations, threeWayMerge, wsErr
}
ctx.Warn(DoltMergeWarningCode, err.Error())
return ws, hasConflictsOrViolations, nil
return ws, hasConflictsOrViolations, threeWayMerge, nil
}
return ws, noConflictsOrViolations, err
return ws, noConflictsOrViolations, fastForwardMerge, err
}
ws, err = executeFFMerge(ctx, dbName, spec.Squash, ws, dbData, spec.MergeC)
return ws, noConflictsOrViolations, err
return ws, noConflictsOrViolations, fastForwardMerge, err
}
dbState, ok, err := sess.LookupDbState(ctx, dbName)
if err != nil {
return ws, noConflictsOrViolations, err
return ws, noConflictsOrViolations, threeWayMerge, err
} else if !ok {
return ws, noConflictsOrViolations, sql.ErrDatabaseNotFound.New(dbName)
return ws, noConflictsOrViolations, threeWayMerge, sql.ErrDatabaseNotFound.New(dbName)
}
ws, err = executeMerge(ctx, spec.Squash, spec.HeadC, spec.MergeC, ws, dbState.EditOpts())
@@ -189,22 +198,22 @@ func mergeIntoWorkingSet(ctx *sql.Context, sess *dsess.DoltSession, roots doltdb
// error message
wsErr := sess.SetWorkingSet(ctx, dbName, ws)
if wsErr != nil {
return ws, hasConflictsOrViolations, wsErr
return ws, hasConflictsOrViolations, threeWayMerge, wsErr
}
ctx.Warn(DoltMergeWarningCode, err.Error())
return ws, hasConflictsOrViolations, nil
return ws, hasConflictsOrViolations, threeWayMerge, nil
} else if err != nil {
return ws, noConflictsOrViolations, err
return ws, noConflictsOrViolations, threeWayMerge, err
}
err = sess.SetWorkingSet(ctx, dbName, ws)
if err != nil {
return ws, noConflictsOrViolations, err
return ws, noConflictsOrViolations, threeWayMerge, err
}
return ws, noConflictsOrViolations, nil
return ws, noConflictsOrViolations, threeWayMerge, nil
}
func abortMerge(ctx *sql.Context, workingSet *doltdb.WorkingSet, roots doltdb.Roots) (*doltdb.WorkingSet, error) {
go/libraries/doltcore/sqle/dfunctions/dolt_pull.go
+21 -18
View File
@@ -68,29 +68,31 @@ func (d DoltPullFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
if err != nil {
return noConflictsOrViolations, err
}
return DoDoltPull(ctx, args)
conflicts, _, err := DoDoltPull(ctx, args)
return conflicts, err
}
func DoDoltPull(ctx *sql.Context, args []string) (int, error) {
// DoDoltPull returns conflicts, fast_forward statuses
func DoDoltPull(ctx *sql.Context, args []string) (int, int, error) {
dbName := ctx.GetCurrentDatabase()
if len(dbName) == 0 {
return noConflictsOrViolations, fmt.Errorf("empty database name.")
return noConflictsOrViolations, threeWayMerge, fmt.Errorf("empty database name.")
}
sess := dsess.DSessFromSess(ctx.Session)
dbData, ok := sess.GetDbData(ctx, dbName)
if !ok {
return noConflictsOrViolations, sql.ErrDatabaseNotFound.New(dbName)
return noConflictsOrViolations, threeWayMerge, sql.ErrDatabaseNotFound.New(dbName)
}
apr, err := cli.CreatePullArgParser().Parse(args)
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
if apr.NArg() > 1 {
return noConflictsOrViolations, actions.ErrInvalidPullArgs
return noConflictsOrViolations, threeWayMerge, actions.ErrInvalidPullArgs
}
var remoteName string
@@ -100,20 +102,21 @@ func DoDoltPull(ctx *sql.Context, args []string) (int, error) {
pullSpec, err := env.NewPullSpec(ctx, dbData.Rsr, remoteName, apr.Contains(cli.SquashParam), apr.Contains(cli.NoFFParam), apr.Contains(cli.ForceFlag))
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
srcDB, err := pullSpec.Remote.GetRemoteDBWithoutCaching(ctx, dbData.Ddb.ValueReadWriter().Format())
if err != nil {
return noConflictsOrViolations, fmt.Errorf("failed to get remote db; %w", err)
return noConflictsOrViolations, threeWayMerge, fmt.Errorf("failed to get remote db; %w", err)
}
ws, err := sess.WorkingSet(ctx, dbName)
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
var conflicts int
var fastForward int
for _, refSpec := range pullSpec.RefSpecs {
remoteTrackRef := refSpec.DestRef(pullSpec.Branch)
@@ -122,42 +125,42 @@ func DoDoltPull(ctx *sql.Context, args []string) (int, error) {
// todo: can we pass nil for either of the channels?
srcDBCommit, err := actions.FetchRemoteBranch(ctx, dbData.Rsw.TempTableFilesDir(), pullSpec.Remote, srcDB, dbData.Ddb, pullSpec.Branch, runProgFuncs, stopProgFuncs)
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
// TODO: this could be replaced with a canFF check to test for error
err = dbData.Ddb.FastForward(ctx, remoteTrackRef, srcDBCommit)
if err != nil {
return noConflictsOrViolations, fmt.Errorf("fetch failed; %w", err)
return noConflictsOrViolations, threeWayMerge, fmt.Errorf("fetch failed; %w", err)
}
roots, ok := sess.GetRoots(ctx, dbName)
if !ok {
return noConflictsOrViolations, sql.ErrDatabaseNotFound.New(dbName)
return noConflictsOrViolations, threeWayMerge, sql.ErrDatabaseNotFound.New(dbName)
}
mergeSpec, err := createMergeSpec(ctx, sess, dbName, apr, remoteTrackRef.String())
if err != nil {
return noConflictsOrViolations, err
return noConflictsOrViolations, threeWayMerge, err
}
ws, conflicts, err = mergeIntoWorkingSet(ctx, sess, roots, ws, dbName, mergeSpec)
ws, conflicts, fastForward, err = mergeIntoWorkingSet(ctx, sess, roots, ws, dbName, mergeSpec)
if err != nil && !errors.Is(doltdb.ErrUpToDate, err) {
return conflicts, err
return conflicts, fastForward, err
}
err = sess.SetWorkingSet(ctx, dbName, ws)
if err != nil {
return conflicts, err
return conflicts, fastForward, err
}
}
}
err = actions.FetchFollowTags(ctx, dbData.Rsw.TempTableFilesDir(), srcDB, dbData.Ddb, runProgFuncs, stopProgFuncs)
if err != nil {
return noConflictsOrViolations, err
return conflicts, fastForward, err
}
return noConflictsOrViolations, nil
return conflicts, fastForward, nil
}
func pullerProgFunc(ctx context.Context, statsCh <-chan pull.Stats) {
go/libraries/doltcore/sqle/dfunctions/merge.go
+2 -1
View File
@@ -43,7 +43,8 @@ func (mf *MergeFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
if err != nil {
return noConflictsOrViolations, err
}
return DoDoltMerge(ctx, args)
hasConflicts, _, err := DoDoltMerge(ctx, args)
return hasConflicts, err
}
// String implements the Stringer interface.
go/libraries/doltcore/sqle/dprocedures/dolt_merge.go
+2 -2
View File
@@ -22,9 +22,9 @@ import (
// doltMerge is the stored procedure version of the functions `merge` and `dolt_merge`.
func doltMerge(ctx *sql.Context, args ...string) (sql.RowIter, error) {
res, err := dfunctions.DoDoltMerge(ctx, args)
hasConflicts, ff, err := dfunctions.DoDoltMerge(ctx, args)
if err != nil {
return nil, err
}
return rowToIter(int64(res)), nil
return rowToIter(int64(ff), int64(hasConflicts)), nil
}
go/libraries/doltcore/sqle/dprocedures/dolt_pull.go
+2 -2
View File
@@ -22,9 +22,9 @@ import (
// doltPull is the stored procedure version of the function `dolt_pull`.
func doltPull(ctx *sql.Context, args ...string) (sql.RowIter, error) {
res, err := dfunctions.DoDoltPull(ctx, args)
conflicts, ff, err := dfunctions.DoDoltPull(ctx, args)
if err != nil {
return nil, err
}
return rowToIter(int64(res)), nil
return rowToIter(int64(ff), int64(conflicts)), nil
}
go/libraries/doltcore/sqle/dprocedures/init.go
+4 -4
View File
@@ -24,8 +24,8 @@ var DoltProcedures = []sql.ExternalStoredProcedureDetails{
{Name: "dolt_clean", Schema: int64Schema("status"), Function: doltClean},
{Name: "dolt_commit", Schema: stringSchema("hash"), Function: doltCommit},
{Name: "dolt_fetch", Schema: int64Schema("success"), Function: doltFetch},
{Name: "dolt_merge", Schema: int64Schema("no_conflicts"), Function: doltMerge},
{Name: "dolt_pull", Schema: int64Schema("no_conflicts"), Function: doltPull},
{Name: "dolt_merge", Schema: int64Schema("fast_forward", "conflicts"), Function: doltMerge},
{Name: "dolt_pull", Schema: int64Schema("fast_forward", "conflicts"), Function: doltPull},
{Name: "dolt_push", Schema: int64Schema("success"), Function: doltPush},
{Name: "dolt_reset", Schema: int64Schema("status"), Function: doltReset},
{Name: "dolt_revert", Schema: int64Schema("status"), Function: doltRevert},
@@ -37,8 +37,8 @@ var DoltProcedures = []sql.ExternalStoredProcedureDetails{
{Name: "dclean", Schema: int64Schema("status"), Function: doltClean},
{Name: "dcommit", Schema: stringSchema("hash"), Function: doltCommit},
{Name: "dfetch", Schema: int64Schema("success"), Function: doltFetch},
{Name: "dmerge", Schema: int64Schema("no_conflicts"), Function: doltMerge},
{Name: "dpull", Schema: int64Schema("no_conflicts"), Function: doltPull},
{Name: "dmerge", Schema: int64Schema("fast_forward", "conflicts"), Function: doltMerge},
{Name: "dpull", Schema: int64Schema("fast_forward", "conflicts"), Function: doltPull},
{Name: "dpush", Schema: int64Schema("success"), Function: doltPush},
{Name: "dreset", Schema: int64Schema("status"), Function: doltReset},
{Name: "drevert", Schema: int64Schema("status"), Function: doltRevert},
go/libraries/doltcore/sqle/dtables/conflicts_tables.go
+5 -4
View File
@@ -29,6 +29,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
"github.com/dolthub/dolt/go/store/pool"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
@@ -158,10 +159,10 @@ func newProllyConflictRowIter(ctx context.Context, conflictMap prolly.ConflictMa
return prollyConflictRowIter{}, err
}
kd := prolly.KeyDescriptorFromSchema(baseSch)
baseVD := prolly.ValueDescriptorFromSchema(baseSch)
oursVD := prolly.ValueDescriptorFromSchema(ourSch)
theirsVD := prolly.ValueDescriptorFromSchema(theirSch)
kd := shim.KeyDescriptorFromSchema(baseSch)
baseVD := shim.ValueDescriptorFromSchema(baseSch)
oursVD := shim.ValueDescriptorFromSchema(ourSch)
theirsVD := shim.ValueDescriptorFromSchema(theirSch)
b := 0
o := kd.Count() + baseVD.Count()
go/libraries/doltcore/sqle/dtables/prolly_row_conv.go
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/rowconv"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/val"
)
@@ -71,7 +71,7 @@ func NewProllyRowConverter(inSch, outSch schema.Schema, warnFn rowconv.WarnFunct
}
}
kd, vd := prolly.MapDescriptorsFromSchema(inSch)
kd, vd := shim.MapDescriptorsFromSchema(inSch)
return ProllyRowConverter{
inSchema: inSch,
outSchema: outSch,
go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go
+4 -4
View File
@@ -305,7 +305,7 @@ func TestDoltUserPrivileges(t *testing.T) {
t.Skip()
}
}
enginetest.RunQueryWithContext(t, engine, ctx, statement)
enginetest.RunQueryWithContext(t, engine, harness, ctx, statement)
}
for _, assertion := range script.Assertions {
if sh, ok := interface{}(harness).(enginetest.SkippingHarness); ok {
@@ -329,11 +329,11 @@ func TestDoltUserPrivileges(t *testing.T) {
if assertion.ExpectedErr != nil {
t.Run(assertion.Query, func(t *testing.T) {
enginetest.AssertErrWithCtx(t, engine, ctx, assertion.Query, assertion.ExpectedErr)
enginetest.AssertErrWithCtx(t, engine, harness, ctx, assertion.Query, assertion.ExpectedErr)
})
} else if assertion.ExpectedErrStr != "" {
t.Run(assertion.Query, func(t *testing.T) {
enginetest.AssertErrWithCtx(t, engine, ctx, assertion.Query, nil, assertion.ExpectedErrStr)
enginetest.AssertErrWithCtx(t, engine, harness, ctx, assertion.Query, nil, assertion.ExpectedErrStr)
})
} else {
t.Run(assertion.Query, func(t *testing.T) {
@@ -710,7 +710,7 @@ func TestSingleTransactionScript(t *testing.T) {
},
{
Query: "/* client b */ call dolt_merge('main')",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{0, 1}},
},
{
Query: "/* client b */ select count(*) from dolt_conflicts",
go/libraries/doltcore/sqle/enginetest/dolt_harness.go
+10
View File
@@ -69,6 +69,7 @@ var _ enginetest.VersionedDBHarness = (*DoltHarness)(nil)
var _ enginetest.ForeignKeyHarness = (*DoltHarness)(nil)
var _ enginetest.KeylessTableHarness = (*DoltHarness)(nil)
var _ enginetest.ReadOnlyDatabaseHarness = (*DoltHarness)(nil)
var _ enginetest.ValidatingHarness = (*DoltHarness)(nil)
func newDoltHarness(t *testing.T) *DoltHarness {
dEnv := dtestutils.CreateTestEnv()
@@ -445,6 +446,15 @@ func (d *DoltHarness) SnapshotTable(db sql.VersionedDatabase, name string, asOf
return nil
}
func (d *DoltHarness) ValidateEngine(ctx *sql.Context, e *gms.Engine) (err error) {
for _, db := range e.Analyzer.Catalog.AllDatabases(ctx) {
if err = ValidateDatabase(ctx, db); err != nil {
return err
}
}
return
}
func dsqleDBsAsSqlDBs(dbs []sqle.Database) []sql.Database {
sqlDbs := make([]sql.Database, 0, len(dbs))
for _, db := range dbs {
go/libraries/doltcore/sqle/enginetest/dolt_queries.go
+79 -46
View File
@@ -736,7 +736,7 @@ var HistorySystemTableScriptTests = []queries.ScriptTest{
var MergeScripts = []queries.ScriptTest{
{
Name: "DOLT_MERGE ff correctly works with autocommit off",
Name: "CALL DOLT_MERGE ff correctly works with autocommit off",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key)",
"INSERT INTO test VALUES (0),(1),(2);",
@@ -751,8 +751,8 @@ var MergeScripts = []queries.ScriptTest{
Assertions: []queries.ScriptTestAssertion{
{
// FF-Merge
Query: "SELECT DOLT_MERGE('feature-branch')",
Expected: []sql.Row{{1}},
Query: "CALL DOLT_MERGE('feature-branch')",
Expected: []sql.Row{{1, 0}},
},
{
Query: "SELECT * from dolt_status",
@@ -769,7 +769,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE no-ff correctly works with autocommit off",
Name: "CALL DOLT_MERGE no-ff correctly works with autocommit off",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key)",
"INSERT INTO test VALUES (0),(1),(2);",
@@ -784,8 +784,8 @@ var MergeScripts = []queries.ScriptTest{
Assertions: []queries.ScriptTestAssertion{
{
// No-FF-Merge
Query: "SELECT DOLT_MERGE('feature-branch', '-no-ff', '-m', 'this is a no-ff')",
Expected: []sql.Row{{1}},
Query: "CALL DOLT_MERGE('feature-branch', '-no-ff', '-m', 'this is a no-ff')",
Expected: []sql.Row{{1, 0}},
},
{
Query: "SELECT * from dolt_status",
@@ -806,7 +806,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE without conflicts correctly works with autocommit off",
Name: "CALL DOLT_MERGE without conflicts correctly works with autocommit off",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key)",
"INSERT INTO test VALUES (0),(1),(2);",
@@ -822,8 +822,8 @@ var MergeScripts = []queries.ScriptTest{
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "SELECT DOLT_MERGE('feature-branch', '-m', 'this is a merge')",
Expected: []sql.Row{{1}},
Query: "CALL DOLT_MERGE('feature-branch', '-m', 'this is a merge')",
Expected: []sql.Row{{0, 0}},
},
{
Query: "SELECT COUNT(*) from dolt_status",
@@ -844,7 +844,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE with conflicts can be correctly resolved when autocommit is off",
Name: "CALL DOLT_MERGE with conflicts can be correctly resolved when autocommit is off",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key, val int)",
"INSERT INTO test VALUES (0, 0)",
@@ -860,8 +860,8 @@ var MergeScripts = []queries.ScriptTest{
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "SELECT DOLT_MERGE('feature-branch', '-m', 'this is a merge')",
Expected: []sql.Row{{0}},
Query: "CALL DOLT_MERGE('feature-branch', '-m', 'this is a merge')",
Expected: []sql.Row{{0, 1}},
},
{
Query: "SELECT * from dolt_status",
@@ -898,7 +898,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE ff & squash correctly works with autocommit off",
Name: "CALL DOLT_MERGE ff & squash correctly works with autocommit off",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key)",
"INSERT INTO test VALUES (0),(1),(2);",
@@ -912,8 +912,8 @@ var MergeScripts = []queries.ScriptTest{
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "SELECT DOLT_MERGE('feature-branch', '--squash')",
Expected: []sql.Row{{1}},
Query: "CALL DOLT_MERGE('feature-branch', '--squash')",
Expected: []sql.Row{{1, 0}},
},
{
Query: "SELECT count(*) from dolt_status",
@@ -930,7 +930,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE ff & squash with a checkout in between",
Name: "CALL DOLT_MERGE ff & squash with a checkout in between",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key)",
"INSERT INTO test VALUES (0),(1),(2);",
@@ -944,8 +944,8 @@ var MergeScripts = []queries.ScriptTest{
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "SELECT DOLT_MERGE('feature-branch', '--squash')",
Expected: []sql.Row{{1}},
Query: "CALL DOLT_MERGE('feature-branch', '--squash')",
Expected: []sql.Row{{1, 0}},
},
{
Query: "SELECT DOLT_CHECKOUT('-b', 'other')",
@@ -958,7 +958,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE ff",
Name: "CALL DOLT_MERGE ff",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key)",
"INSERT INTO test VALUES (0),(1),(2);",
@@ -972,8 +972,8 @@ var MergeScripts = []queries.ScriptTest{
Assertions: []queries.ScriptTestAssertion{
{
// FF-Merge
Query: "SELECT DOLT_MERGE('feature-branch')",
Expected: []sql.Row{{1}},
Query: "CALL DOLT_MERGE('feature-branch')",
Expected: []sql.Row{{1, 0}},
},
{
Query: "SELECT * from dolt_status",
@@ -990,7 +990,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE no-ff",
Name: "CALL DOLT_MERGE no-ff",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key)",
"INSERT INTO test VALUES (0),(1),(2);",
@@ -1004,8 +1004,8 @@ var MergeScripts = []queries.ScriptTest{
Assertions: []queries.ScriptTestAssertion{
{
// No-FF-Merge
Query: "SELECT DOLT_MERGE('feature-branch', '-no-ff', '-m', 'this is a no-ff')",
Expected: []sql.Row{{1}},
Query: "CALL DOLT_MERGE('feature-branch', '-no-ff', '-m', 'this is a no-ff')",
Expected: []sql.Row{{1, 0}},
},
{
Query: "SELECT * from dolt_status",
@@ -1026,7 +1026,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE with no conflicts works",
Name: "CALL DOLT_MERGE with no conflicts works",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key)",
"INSERT INTO test VALUES (0),(1),(2);",
@@ -1041,8 +1041,8 @@ var MergeScripts = []queries.ScriptTest{
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "SELECT DOLT_MERGE('feature-branch', '-m', 'this is a merge')",
Expected: []sql.Row{{1}},
Query: "CALL DOLT_MERGE('feature-branch', '-m', 'this is a merge')",
Expected: []sql.Row{{0, 0}},
},
{
Query: "SELECT COUNT(*) from dolt_status",
@@ -1063,7 +1063,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE with conflict is queryable and committable with dolt_allow_commit_conflicts on",
Name: "CALL DOLT_MERGE with conflict is queryable and committable with dolt_allow_commit_conflicts on",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key, val int)",
"INSERT INTO test VALUES (0, 0)",
@@ -1079,8 +1079,8 @@ var MergeScripts = []queries.ScriptTest{
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "SELECT DOLT_MERGE('feature-branch')",
Expected: []sql.Row{{0}},
Query: "CALL DOLT_MERGE('feature-branch')",
Expected: []sql.Row{{0, 1}},
},
{
Query: "SELECT count(*) from dolt_conflicts_test",
@@ -1088,7 +1088,7 @@ var MergeScripts = []queries.ScriptTest{
},
{
Query: "SELECT DOLT_MERGE('--abort')",
Expected: []sql.Row{{1}},
Expected: []sql.Row{{0}},
},
{
Query: "SELECT * FROM test",
@@ -1117,7 +1117,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE with conflicts can be aborted when autocommit is off",
Name: "CALL DOLT_MERGE with conflicts can be aborted when autocommit is off",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key, val int)",
"INSERT INTO test VALUES (0, 0)",
@@ -1133,8 +1133,8 @@ var MergeScripts = []queries.ScriptTest{
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "SELECT DOLT_MERGE('feature-branch', '-m', 'this is a merge')",
Expected: []sql.Row{{0}},
Query: "CALL DOLT_MERGE('feature-branch', '-m', 'this is a merge')",
Expected: []sql.Row{{0, 1}},
},
{
Query: "SELECT * from dolt_status",
@@ -1146,7 +1146,7 @@ var MergeScripts = []queries.ScriptTest{
},
{
Query: "SELECT DOLT_MERGE('--abort')",
Expected: []sql.Row{{1}},
Expected: []sql.Row{{0}},
},
{
Query: "SELECT * from dolt_status",
@@ -1167,7 +1167,7 @@ var MergeScripts = []queries.ScriptTest{
},
},
{
Name: "DOLT_MERGE complains when a merge overrides local changes",
Name: "CALL DOLT_MERGE complains when a merge overrides local changes",
SetUpScript: []string{
"CREATE TABLE test (pk int primary key, val int)",
"INSERT INTO test VALUES (0, 0)",
@@ -1182,7 +1182,7 @@ var MergeScripts = []queries.ScriptTest{
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "SELECT DOLT_MERGE('feature-branch', '-m', 'this is a merge')",
Query: "CALL DOLT_MERGE('feature-branch', '-m', 'this is a merge')",
ExpectedErr: dfunctions.ErrUncommittedChanges,
},
},
@@ -1206,7 +1206,7 @@ var MergeScripts = []queries.ScriptTest{
Assertions: []queries.ScriptTestAssertion{
{
Query: "call dolt_merge('b1')",
Expected: []sql.Row{{1}},
Expected: []sql.Row{{0, 0}},
},
{
Query: "select count(*) from dolt_conflicts",
@@ -1259,7 +1259,7 @@ var MergeScripts = []queries.ScriptTest{
},
{
Query: "CALL DOLT_MERGE('branch1');",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{0, 1}},
},
{
Query: "SELECT violation_type, pk, parent_fk from dolt_constraint_violations_child;",
@@ -1271,7 +1271,7 @@ var MergeScripts = []queries.ScriptTest{
},
{
Query: "CALL DOLT_COMMIT('-am', 'commit constraint violations');",
ExpectedErrStr: "error: the table(s) child has constraint violations",
ExpectedErrStr: "error: the table(s) child have constraint violations",
},
{
Query: "CALL DOLT_COMMIT('-afm', 'commit constraint violations');",
@@ -1307,7 +1307,7 @@ var MergeScripts = []queries.ScriptTest{
},
{
Query: "CALL DOLT_MERGE('branch2');",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{0, 1}},
},
{
Query: "SELECT violation_type, pk, parent_fk from dolt_constraint_violations_child;",
@@ -1319,7 +1319,7 @@ var MergeScripts = []queries.ScriptTest{
},
{
Query: "CALL DOLT_COMMIT('-am', 'commit non-conflicting merge');",
ExpectedErrStr: "error: the table(s) child has constraint violations",
ExpectedErrStr: "error: the table(s) child have constraint violations",
},
{
Query: "CALL DOLT_COMMIT('-afm', 'commit non-conflicting merge');",
@@ -1343,7 +1343,7 @@ var MergeScripts = []queries.ScriptTest{
},
{
Query: "CALL DOLT_MERGE('branch3');",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{0, 1}},
},
{
Query: "SELECT violation_type, pk, parent_fk from dolt_constraint_violations_child;",
@@ -1379,7 +1379,7 @@ var MergeScripts = []queries.ScriptTest{
},
{
Query: "CALL DOLT_MERGE('other');",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{0, 1}},
},
{
Query: "SELECT * from parent;",
@@ -1452,7 +1452,7 @@ var MergeScripts = []queries.ScriptTest{
},
{
Query: "CALL DOLT_MERGE('other');",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{0, 1}},
},
{
Query: "SELECT * from parent;",
@@ -1477,7 +1477,7 @@ var MergeScripts = []queries.ScriptTest{
},
{
Query: "CALL DOLT_MERGE('other2');",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{0, 1}},
},
{
Query: "SELECT * from parent;",
@@ -1635,6 +1635,39 @@ var DoltBranchScripts = []queries.ScriptTest{
},
},
},
{
Name: "Create branch from startpoint",
SetUpScript: []string{
"create table a (x int)",
"set @commit1 = (select DOLT_COMMIT('-am', 'add table a'));",
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "show tables",
Expected: []sql.Row{{"a"}, {"myview"}},
},
{
Query: "CALL DOLT_CHECKOUT('-b', 'newBranch', 'head~1')",
Expected: []sql.Row{{0}},
},
{
Query: "show tables",
Expected: []sql.Row{{"myview"}},
},
{
Query: "CALL DOLT_CHECKOUT('-b', 'newBranch2', @commit1)",
Expected: []sql.Row{{0}},
},
{
Query: "show tables",
Expected: []sql.Row{{"a"}, {"myview"}},
},
{
Query: "CALL DOLT_CHECKOUT('-b', 'otherBranch', 'unknownCommit')",
ExpectedErrStr: "fatal: 'unknownCommit' is not a commit and a branch 'otherBranch' cannot be created from it",
},
},
},
}
var DoltReset = []queries.ScriptTest{
go/libraries/doltcore/sqle/enginetest/dolt_transaction_queries.go
+5 -5
View File
@@ -953,7 +953,7 @@ var DoltConflictHandlingTests = []queries.TransactionTest{
},
{
Query: "/* client b */ call dolt_merge('main')",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{0, 1}},
},
{
Query: "/* client b */ select count(*) from dolt_conflicts",
@@ -1043,7 +1043,7 @@ var DoltConflictHandlingTests = []queries.TransactionTest{
},
{
Query: "/* client b */ call dolt_merge('main')",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{0, 1}},
},
{
Query: "/* client b */ select count(*) from dolt_conflicts",
@@ -1131,7 +1131,7 @@ var DoltConflictHandlingTests = []queries.TransactionTest{
},
{
Query: "/* client b */ call dolt_merge('main')",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{0, 1}},
},
{
Query: "/* client b */ select count(*) from dolt_conflicts",
@@ -1183,7 +1183,7 @@ var DoltSqlFuncTransactionTests = []queries.TransactionTest{
},
{
Query: "/* client a */ SELECT DOLT_MERGE('feature-branch')",
Expected: []sql.Row{{0}},
Expected: []sql.Row{{1}},
},
{
Query: "/* client a */ SELECT count(*) from dolt_conflicts_test",
@@ -1215,7 +1215,7 @@ var DoltSqlFuncTransactionTests = []queries.TransactionTest{
},
{
Query: "/* client a */ SELECT DOLT_MERGE('--abort')",
Expected: []sql.Row{{1}},
Expected: []sql.Row{{0}},
},
{
Query: "/* client a */ commit",
go/libraries/doltcore/sqle/enginetest/validation.go
+246
View File
@@ -0,0 +1,246 @@
// Copyright 2020 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package enginetest
import (
"context"
"fmt"
"io"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/go-mysql-server/sql/mysql_db"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
func ValidateDatabase(ctx context.Context, db sql.Database) (err error) {
switch tdb := db.(type) {
case sqle.Database:
return ValidateDoltDatabase(ctx, tdb)
case mysql_db.PrivilegedDatabase:
return ValidateDatabase(ctx, tdb.Unwrap())
default:
return nil
}
}
func ValidateDoltDatabase(ctx context.Context, db sqle.Database) (err error) {
if !types.IsFormat_DOLT_1(db.GetDoltDB().Format()) {
return nil
}
for _, stage := range validationStages {
if err = stage(ctx, db); err != nil {
return err
}
}
return
}
type validator func(ctx context.Context, db sqle.Database) error
var validationStages = []validator{
validateChunkReferences,
validateSecondaryIndexes,
}
// validateChunkReferences checks for dangling chunks.
func validateChunkReferences(ctx context.Context, db sqle.Database) error {
validateIndex := func(ctx context.Context, idx durable.Index) error {
pm := durable.ProllyMapFromIndex(idx)
return pm.WalkNodes(ctx, func(ctx context.Context, nd tree.Node) error {
if nd.Size() <= 0 {
return fmt.Errorf("encountered nil tree.Node")
}
return nil
})
}
cb := func(n string, t *doltdb.Table, sch schema.Schema) (stop bool, err error) {
if sch == nil {
return true, fmt.Errorf("expected non-nil schema: %v", sch)
}
rows, err := t.GetRowData(ctx)
if err != nil {
return true, err
}
if err = validateIndex(ctx, rows); err != nil {
return true, err
}
indexes, err := t.GetIndexSet(ctx)
if err != nil {
return true, err
}
err = durable.IterAllIndexes(ctx, sch, indexes, func(_ string, idx durable.Index) error {
return validateIndex(ctx, idx)
})
if err != nil {
return true, err
}
return
}
return iterDatabaseTables(ctx, db, cb)
}
// validateSecondaryIndexes checks that secondary index contents are consistent
// with primary index contents.
func validateSecondaryIndexes(ctx context.Context, db sqle.Database) error {
cb := func(n string, t *doltdb.Table, sch schema.Schema) (stop bool, err error) {
rows, err := t.GetRowData(ctx)
if err != nil {
return false, err
}
primary := durable.ProllyMapFromIndex(rows)
for _, def := range sch.Indexes().AllIndexes() {
set, err := t.GetIndexSet(ctx)
if err != nil {
return true, err
}
idx, err := set.GetIndex(ctx, sch, def.Name())
if err != nil {
return true, err
}
secondary := durable.ProllyMapFromIndex(idx)
err = validateIndexConsistency(ctx, sch, def, primary, secondary)
if err != nil {
return true, err
}
}
return false, nil
}
return iterDatabaseTables(ctx, db, cb)
}
func validateIndexConsistency(
ctx context.Context,
sch schema.Schema,
def schema.Index,
primary, secondary prolly.Map,
) error {
// secondary indexes have empty keys
idxDesc, _ := secondary.Descriptors()
builder := val.NewTupleBuilder(idxDesc)
mapping := ordinalMappingsForSecondaryIndex(sch, def)
kd, _ := primary.Descriptors()
pkSize := kd.Count()
iter, err := primary.IterAll(ctx)
if err != nil {
return err
}
for {
key, value, err := iter.Next(ctx)
if err == io.EOF {
return nil
}
if err != nil {
return err
}
// make secondary index key
for i := range mapping {
j := mapping.MapOrdinal(i)
if j < pkSize {
builder.PutRaw(i, key.GetField(j))
} else {
builder.PutRaw(i, value.GetField(j-pkSize))
}
}
k := builder.Build(primary.Pool())
ok, err := secondary.Has(ctx, k)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("index key %v not found in index %s", k, def.Name())
}
}
}
func ordinalMappingsForSecondaryIndex(sch schema.Schema, def schema.Index) (ord val.OrdinalMapping) {
// assert empty values for secondary indexes
if def.Schema().GetNonPKCols().Size() > 0 {
panic("expected empty secondary index values")
}
secondary := def.Schema().GetPKCols()
ord = make(val.OrdinalMapping, secondary.Size())
for i := range ord {
name := secondary.GetAtIndex(i).Name
ord[i] = -1
pks := sch.GetPKCols().GetColumns()
for j, col := range pks {
if col.Name == name {
ord[i] = j
}
}
vals := sch.GetNonPKCols().GetColumns()
for j, col := range vals {
if col.Name == name {
ord[i] = j + len(pks)
}
}
if ord[i] < 0 {
panic("column " + name + " not found")
}
}
return
}
// iterDatabaseTables is a utility to factor out common validation access patterns.
func iterDatabaseTables(
ctx context.Context,
db sqle.Database,
cb func(name string, t *doltdb.Table, sch schema.Schema) (bool, error),
) error {
ddb := db.GetDoltDB()
branches, err := ddb.GetBranches(ctx)
if err != nil {
return err
}
for i := range branches {
var c *doltdb.Commit
var r *doltdb.RootValue
c, err = ddb.ResolveCommitRef(ctx, branches[i])
if err != nil {
return err
}
r, err = c.GetRootValue(ctx)
if err != nil {
return err
}
if err = r.IterTables(ctx, cb); err != nil {
return err
}
}
return nil
}
go/libraries/doltcore/sqle/index/prolly_fields.go
+14
View File
@@ -16,7 +16,9 @@ package index
import (
"encoding/json"
"errors"
"fmt"
"math"
"time"
"github.com/dolthub/go-mysql-server/sql"
@@ -28,6 +30,8 @@ import (
"github.com/dolthub/dolt/go/store/val"
)
var ErrValueExceededMaxFieldSize = errors.New("value exceeded max field size of 65kb")
// todo(andy): this should go in GMS
func DenormalizeRow(sch sql.Schema, row sql.Row) (sql.Row, error) {
var err error
@@ -201,13 +205,23 @@ func PutField(tb *val.TupleBuilder, i int, v interface{}) error {
tb.PutString(i, v.(string))
case val.ByteStringEnc:
if s, ok := v.(string); ok {
if len(s) > math.MaxUint16 {
return ErrValueExceededMaxFieldSize
}
v = []byte(s)
}
tb.PutByteString(i, v.([]byte))
case val.GeometryEnc:
geo := serializeGeometry(v)
if len(geo) > math.MaxUint16 {
return ErrValueExceededMaxFieldSize
}
tb.PutGeometry(i, serializeGeometry(v))
case val.JSONEnc:
buf, err := convJson(v)
if len(buf) > math.MaxUint16 {
return ErrValueExceededMaxFieldSize
}
if err != nil {
return err
}
go/libraries/doltcore/sqle/sqlselect_test.go
+1 -1
View File
@@ -88,7 +88,7 @@ func LoadedLocalLocation() *time.Location {
func BasicSelectTests() []SelectTest {
headCommitHash := "73hc2robs4v0kt9taoe3m5hd49dmrgun"
if types.Format_Default == types.Format_DOLT_DEV {
headCommitHash = "8hrmmj29n9keblqs7r9423f69uesfgn6"
headCommitHash = "hqcfvhaumjtetijsqdnnjhj87f0lesbo"
}
return []SelectTest{
{
go/serial/prolly.fbs
-16
View File
@@ -52,22 +52,6 @@ table ProllyTreeNode {
tree_level:uint8;
}
/// Refmap has been deprecated in favor of AddressMap
table RefMap {
// map keys ordered lexigraphically
names:[string] (required);
// array of chunk addresses
// - subtree addresses for internal prolly tree nodes
// - value addresses for AddressMap leaf nodes
ref_array:[ubyte] (required);
// total count of prolly tree
tree_count:uint64;
// prolly tree level, 0 for leaf nodes
tree_level:uint8;
}
table CommitClosure {
// array of commit ref hashes
ref_array:[ubyte] (required);
go/serial/rootvalue.fbs
+2 -1
View File
@@ -19,7 +19,8 @@ namespace serial;
table RootValue {
feature_version:int64;
tables:RefMap (required);
tables:[ubyte]; // Serialized AddressMap.
foreign_key_addr:[ubyte];
super_schemas_addr:[ubyte];
}
go/serial/storeroot.fbs
+1 -3
View File
@@ -19,9 +19,7 @@ namespace serial;
// StoreRoot is the top-level chunk in the store,
// containing references to all named Refs.
table StoreRoot {
// map from ref name to ref hash
// with map root node inlined
refs:RefMap (required);
address_map:[ubyte]; // Embedded serialized AddressMap.
}
// KEEP THIS IN SYNC WITH fileidentifiers.go
go/serial/table.fbs
+1 -1
View File
@@ -27,7 +27,7 @@ table Table {
// Entries map from index names to addresses of
// index maps.
secondary_indexes:RefMap (required);
secondary_indexes:[ubyte]; // Embedded AddressMap
auto_increment_value:uint64;
go/store/cmd/noms/noms_root_test.go
+2 -2
View File
@@ -55,8 +55,8 @@ func (s *nomsRootTestSuite) TestBasic() {
goldenHello := "u8g2r4qg97kkqn42lvao77st2mv3bpl0\n"
goldenGoodbye := "70b9adi6amrab3a5t4hcibdob0cq49m0\n"
if types.Format_Default == types.Format_DOLT_DEV {
goldenHello = "bu6q8qir2vfq6lliqrko4jqls0rjga0h\n"
goldenGoodbye = "79a0kfbq40fl8s359e8ssitevt76jsnv\n"
goldenHello = "jeps3m1q780r3cv8tgfv7658ft9cbt24\n"
goldenGoodbye = "5cu08e3onbbna253menst2koc9sq5q0l\n"
}
ds, _ = datas.CommitValue(context.Background(), db, ds, types.String("hello!"))
go/store/cmd/noms/noms_show.go
+57 -83
View File
@@ -27,7 +27,6 @@ import (
"fmt"
"io"
"os"
"strings"
flag "github.com/juju/gnuflag"
@@ -35,7 +34,7 @@ import (
"github.com/dolthub/dolt/go/store/cmd/noms/util"
"github.com/dolthub/dolt/go/store/config"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/util/datetime"
@@ -73,23 +72,12 @@ func setupShowFlags() *flag.FlagSet {
func runShow(ctx context.Context, args []string) int {
cfg := config.NewResolver()
var value interface{}
var value types.Value
database, vrw, value, err := cfg.GetPath(ctx, args[0])
if err != nil && strings.Contains(err.Error(), "unknown type") {
// If noms can't decode a value but it does exist, we'll assume it's a prolly node and treat it as such
sp, err := cfg.GetDatabaseSpecForPath(ctx, args[0])
if err != nil {
util.CheckErrorNoUsage(err)
database = sp.GetDatabase(ctx)
vrw = sp.GetVRW(ctx)
cs := sp.NewChunkStore(ctx)
chunk, err := cs.Get(ctx, sp.Path.Hash)
util.CheckErrorNoUsage(err)
value = tree.NodeFromBytes(chunk.Data())
} else {
util.CheckErrorNoUsage(err)
}
defer database.Close()
@@ -135,97 +123,83 @@ func runShow(ctx context.Context, args []string) int {
return 0
}
func outputType(value interface{}) {
func outputType(value types.Value) {
var typeString string
switch value := value.(type) {
case tree.Node:
typeString = "prolly.Node"
case types.Value:
switch value := value.(type) {
case types.SerialMessage:
switch serial.GetFileID(value) {
case serial.StoreRootFileID:
typeString = "StoreRoot"
case serial.TagFileID:
typeString = "Tag"
case serial.WorkingSetFileID:
typeString = "WorkingSet"
case serial.CommitFileID:
typeString = "Commit"
case serial.RootValueFileID:
typeString = "RootValue"
case serial.TableFileID:
typeString = "TableFile"
case serial.ProllyTreeNodeFileID:
typeString = "ProllyTreeNode"
case serial.AddressMapFileID:
typeString = "AddressMap"
default:
t, err := types.TypeOf(value)
typeString = t.HumanReadableString()
util.CheckError(err)
}
case types.SerialMessage:
switch serial.GetFileID(value) {
case serial.StoreRootFileID:
typeString = "StoreRoot"
case serial.TagFileID:
typeString = "Tag"
case serial.WorkingSetFileID:
typeString = "WorkingSet"
case serial.CommitFileID:
typeString = "Commit"
case serial.RootValueFileID:
typeString = "RootValue"
case serial.TableFileID:
typeString = "TableFile"
case serial.ProllyTreeNodeFileID:
typeString = "ProllyTreeNode"
case serial.AddressMapFileID:
typeString = "AddressMap"
default:
t, err := types.TypeOf(value)
util.CheckErrorNoUsage(err)
typeString = t.HumanReadableString()
util.CheckError(err)
}
default:
typeString = fmt.Sprintf("unknown type %T", value)
t, err := types.TypeOf(value)
util.CheckErrorNoUsage(err)
typeString = t.HumanReadableString()
}
fmt.Fprint(os.Stdout, typeString, " - ")
}
func outputEncodedValue(ctx context.Context, w io.Writer, value interface{}) error {
func outputEncodedValue(ctx context.Context, w io.Writer, value types.Value) error {
switch value := value.(type) {
case types.TupleRowStorage:
node := prolly.NodeFromValue(value)
node := shim.NodeFromValue(value)
return tree.OutputProllyNode(w, node)
case tree.Node:
return tree.OutputProllyNode(w, value)
case types.Value:
switch value := value.(type) {
// Some types of serial message need to be output here because of dependency cycles between type / tree package
case types.SerialMessage:
switch serial.GetFileID(value) {
case serial.TableFileID:
msg := serial.GetRootAsTable(value, 0)
// Some types of serial message need to be output here because of dependency cycles between types / tree package
case types.SerialMessage:
switch serial.GetFileID(value) {
case serial.TableFileID:
msg := serial.GetRootAsTable(value, 0)
fmt.Fprintf(w, "{\n")
fmt.Fprintf(w, "\tSchema: #%s\n", hash.New(msg.SchemaBytes()).String())
fmt.Fprintf(w, "\tViolations: #%s\n", hash.New(msg.ViolationsBytes()).String())
// TODO: merge conflicts, not stable yet
fmt.Fprintf(w, "{\n")
fmt.Fprintf(w, "\tSchema: #%s\n", hash.New(msg.SchemaBytes()).String())
fmt.Fprintf(w, "\tViolations: #%s\n", hash.New(msg.ViolationsBytes()).String())
// TODO: merge conflicts, not stable yet
fmt.Fprintf(w, "\tAutoinc: %d\n", msg.AutoIncrementValue())
fmt.Fprintf(w, "\tAutoinc: %d\n", msg.AutoIncrementValue())
fmt.Fprintf(w, "\tPrimary index: {\n")
node := tree.NodeFromBytes(msg.PrimaryIndexBytes())
tree.OutputProllyNode(w, node)
fmt.Fprintf(w, "\t}\n")
fmt.Fprintf(w, "\tPrimary index: {\n")
node := tree.NodeFromBytes(msg.PrimaryIndexBytes())
tree.OutputProllyNode(w, node)
fmt.Fprintf(w, "\t}\n")
fmt.Fprintf(w, "\tSecondary indexes: {\n")
idxRefs := msg.SecondaryIndexes(nil)
hashes := idxRefs.RefArrayBytes()
for i := 0; i < idxRefs.NamesLength(); i++ {
name := idxRefs.Names(i)
addr := hash.New(hashes[i*20 : (i+1)*20])
fmt.Fprintf(w, "\t\t%s: #%s\n", name, addr.String())
}
fmt.Fprintf(w, "\t}\n")
fmt.Fprintf(w, "}")
fmt.Fprintf(w, "\tSecondary indexes: {\n")
return nil
case serial.ProllyTreeNodeFileID:
node := prolly.NodeFromValue(value)
return tree.OutputProllyNode(w, node)
default:
return types.WriteEncodedValue(ctx, w, value)
node = tree.NodeFromBytes(msg.SecondaryIndexesBytes())
err := tree.OutputAddressMapNode(w, node)
if err != nil {
return err
}
fmt.Fprintf(w, "\t}\n")
fmt.Fprintf(w, "}")
return nil
case serial.StoreRootFileID:
msg := serial.GetRootAsStoreRoot(value, 0)
ambytes := msg.AddressMapBytes()
node := tree.NodeFromBytes(ambytes)
return tree.OutputAddressMapNode(w, node)
default:
return types.WriteEncodedValue(ctx, w, value)
}
default:
_, err := w.Write([]byte(fmt.Sprintf("unknown value type %T: %v", value, value)))
return err
return types.WriteEncodedValue(ctx, w, value)
}
}
go/store/datas/database_common.go
+102 -54
View File
@@ -28,6 +28,8 @@ import (
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -87,17 +89,17 @@ func (db *database) loadDatasetsNomsMap(ctx context.Context, rootHash hash.Hash)
return val.(types.Map), nil
}
func (db *database) loadDatasetsRefmap(ctx context.Context, rootHash hash.Hash) (refmap, error) {
func (db *database) loadDatasetsRefmap(ctx context.Context, rootHash hash.Hash) (prolly.AddressMap, error) {
if rootHash == (hash.Hash{}) {
return empty_refmap(), nil
return prolly.NewEmptyAddressMap(tree.NewNodeStore(db.chunkStore())), nil
}
val, err := db.ReadValue(ctx, rootHash)
if err != nil {
return refmap{}, err
return prolly.AddressMap{}, err
}
return parse_storeroot([]byte(val.(types.SerialMessage))), nil
return parse_storeroot([]byte(val.(types.SerialMessage)), db.chunkStore()), nil
}
func getParentsClosure(ctx context.Context, vrw types.ValueReadWriter, parentRefsL types.List) (types.Ref, bool, error) {
@@ -240,23 +242,15 @@ func getParentsClosure(ctx context.Context, vrw types.ValueReadWriter, parentRef
}
type refmapDatasetsMap struct {
rm refmap
am prolly.AddressMap
}
func (m refmapDatasetsMap) Len() uint64 {
return m.rm.len()
return uint64(m.am.Count())
}
func (m refmapDatasetsMap) IterAll(ctx context.Context, cb func(string, hash.Hash) error) error {
addrs := m.rm.RefMap.RefArrayBytes()
for i := 0; i < m.rm.RefMap.NamesLength(); i++ {
name := string(m.rm.RefMap.Names(i))
addr := hash.New(addrs[i*20 : i*20+20])
if err := cb(name, addr); err != nil {
return err
}
}
return nil
return m.am.IterAll(ctx, cb)
}
type nomsDatasetsMap struct {
@@ -332,7 +326,10 @@ func (db *database) datasetFromMap(ctx context.Context, datasetID string, dsmap
return newDataset(db, datasetID, head, headAddr)
} else if rmdsmap, ok := dsmap.(refmapDatasetsMap); ok {
var err error
curr := rmdsmap.rm.lookup(datasetID)
curr, err := rmdsmap.am.Get(ctx, datasetID)
if err != nil {
return Dataset{}, err
}
var head types.Value
if !curr.IsEmpty() {
head, err = db.ReadValue(ctx, curr)
@@ -436,23 +433,31 @@ func (db *database) doSetHead(ctx context.Context, ds Dataset, addr hash.Hash) e
}
return datasets.Edit().Set(key, ref).Map(ctx)
}, func(ctx context.Context, rm refmap) (refmap, error) {
curr := rm.lookup(ds.ID())
}, func(ctx context.Context, am prolly.AddressMap) (prolly.AddressMap, error) {
curr, err := am.Get(ctx, ds.ID())
if err != nil {
return prolly.AddressMap{}, err
}
if curr != (hash.Hash{}) {
currHead, err := db.readHead(ctx, curr)
if err != nil {
return refmap{}, err
return prolly.AddressMap{}, err
}
currType := currHead.TypeName()
if currType != headType {
return refmap{}, fmt.Errorf("cannot change type of head; currently points at %s but new value would point at %s", currType, headType)
return prolly.AddressMap{}, fmt.Errorf("cannot change type of head; currently points at %s but new value would point at %s", currType, headType)
}
}
h, err := newVal.Hash(db.Format())
if err != nil {
return refmap{}, err
return prolly.AddressMap{}, err
}
return rm.set(ds.ID(), h), nil
ae := am.Editor()
err = ae.Update(ctx, ds.ID(), h)
if err != nil {
return prolly.AddressMap{}, err
}
return ae.Flush(ctx)
})
}
@@ -567,21 +572,29 @@ func (db *database) doCommit(ctx context.Context, datasetID string, datasetCurre
}
return datasets.Edit().Set(types.String(datasetID), newCommitValueRef).Map(ctx)
}, func(ctx context.Context, rm refmap) (refmap, error) {
curr := rm.lookup(datasetID)
}, func(ctx context.Context, am prolly.AddressMap) (prolly.AddressMap, error) {
curr, err := am.Get(ctx, datasetID)
if err != nil {
return prolly.AddressMap{}, err
}
if curr != datasetCurrentAddr {
return refmap{}, ErrMergeNeeded
return prolly.AddressMap{}, ErrMergeNeeded
}
h, err := newCommitValue.Hash(db.Format())
if err != nil {
return refmap{}, err
return prolly.AddressMap{}, err
}
if curr != (hash.Hash{}) {
if curr == h {
return refmap{}, ErrAlreadyCommitted
return prolly.AddressMap{}, ErrAlreadyCommitted
}
}
return rm.set(datasetID, h), nil
ae := am.Editor()
err = ae.Update(ctx, datasetID, h)
if err != nil {
return prolly.AddressMap{}, err
}
return ae.Flush(ctx)
})
}
@@ -616,12 +629,20 @@ func (db *database) doTag(ctx context.Context, datasetID string, tagAddr hash.Ha
}
return datasets.Edit().Set(types.String(datasetID), tagRef).Map(ctx)
}, func(ctx context.Context, rm refmap) (refmap, error) {
curr := rm.lookup(datasetID)
if curr != (hash.Hash{}) {
return refmap{}, fmt.Errorf("tag %s already exists and cannot be altered after creation", datasetID)
}, func(ctx context.Context, am prolly.AddressMap) (prolly.AddressMap, error) {
curr, err := am.Get(ctx, datasetID)
if err != nil {
return prolly.AddressMap{}, err
}
return rm.set(datasetID, tagAddr), nil
if curr != (hash.Hash{}) {
return prolly.AddressMap{}, fmt.Errorf("tag %s already exists and cannot be altered after creation", datasetID)
}
ae := am.Editor()
err = ae.Update(ctx, datasetID, tagAddr)
if err != nil {
return prolly.AddressMap{}, err
}
return ae.Flush(ctx)
})
}
@@ -655,12 +676,20 @@ func (db *database) doUpdateWorkingSet(ctx context.Context, datasetID string, ad
}
return datasets.Edit().Set(types.String(datasetID), ref).Map(ctx)
}, func(ctx context.Context, rm refmap) (refmap, error) {
curr := rm.lookup(datasetID)
if curr != currHash {
return refmap{}, ErrOptimisticLockFailed
}, func(ctx context.Context, am prolly.AddressMap) (prolly.AddressMap, error) {
curr, err := am.Get(ctx, datasetID)
if err != nil {
return prolly.AddressMap{}, err
}
return rm.set(datasetID, addr), nil
if curr != currHash {
return prolly.AddressMap{}, ErrOptimisticLockFailed
}
ae := am.Editor()
err = ae.Update(ctx, datasetID, addr)
if err != nil {
return prolly.AddressMap{}, err
}
return ae.Flush(ctx)
})
}
@@ -739,20 +768,31 @@ func (db *database) CommitWithWorkingSet(
Set(types.String(workingSetDS.ID()), wsValRef).
Set(types.String(commitDS.ID()), commitValRef).
Map(ctx)
}, func(ctx context.Context, rm refmap) (refmap, error) {
currWS := rm.lookup(workingSetDS.ID())
}, func(ctx context.Context, am prolly.AddressMap) (prolly.AddressMap, error) {
currWS, err := am.Get(ctx, workingSetDS.ID())
if err != nil {
return prolly.AddressMap{}, err
}
if currWS != prevWsHash {
return refmap{}, ErrOptimisticLockFailed
return prolly.AddressMap{}, ErrOptimisticLockFailed
}
currDS, err := am.Get(ctx, commitDS.ID())
if err != nil {
return prolly.AddressMap{}, err
}
currDS := rm.lookup(commitDS.ID())
if currDS != currDSHash {
return refmap{}, ErrMergeNeeded
return prolly.AddressMap{}, ErrMergeNeeded
}
rm = rm.edit([]RefMapEdit{
{commitDS.ID(), commitValRef.TargetHash()},
{workingSetDS.ID(), wsAddr},
})
return rm, nil
ae := am.Editor()
err = ae.Update(ctx, commitDS.ID(), commitValRef.TargetHash())
if err != nil {
return prolly.AddressMap{}, err
}
err = ae.Update(ctx, workingSetDS.ID(), wsAddr)
if err != nil {
return prolly.AddressMap{}, err
}
return ae.Flush(ctx)
})
if err != nil {
@@ -783,7 +823,7 @@ func (db *database) Delete(ctx context.Context, ds Dataset) (Dataset, error) {
func (db *database) update(ctx context.Context,
edit func(context.Context, types.Map) (types.Map, error),
editFB func(context.Context, refmap) (refmap, error)) error {
editFB func(context.Context, prolly.AddressMap) (prolly.AddressMap, error)) error {
var (
err error
root hash.Hash
@@ -808,7 +848,7 @@ func (db *database) update(ctx context.Context,
return err
}
data := datasets.storeroot_flatbuffer()
data := storeroot_flatbuffer(datasets)
r, err := db.WriteValue(ctx, types.SerialMessage(data))
if err != nil {
return err
@@ -862,15 +902,23 @@ func (db *database) doDelete(ctx context.Context, datasetIDstr string) error {
return types.Map{}, ErrMergeNeeded
}
return datasets.Edit().Remove(datasetID).Map(ctx)
}, func(ctx context.Context, rm refmap) (refmap, error) {
curr := rm.lookup(datasetIDstr)
}, func(ctx context.Context, am prolly.AddressMap) (prolly.AddressMap, error) {
curr, err := am.Get(ctx, datasetIDstr)
if err != nil {
return prolly.AddressMap{}, err
}
if curr != (hash.Hash{}) && firstHash == (hash.Hash{}) {
firstHash = curr
}
if curr != firstHash {
return refmap{}, ErrMergeNeeded
return prolly.AddressMap{}, ErrMergeNeeded
}
return rm.delete(datasetIDstr), nil
ae := am.Editor()
err = ae.Delete(ctx, datasetIDstr)
if err != nil {
return prolly.AddressMap{}, err
}
return ae.Flush(ctx)
})
}
go/store/datas/refmap.go
+12 -153
View File
@@ -16,173 +16,32 @@ package datas
import (
"bytes"
"sort"
flatbuffers "github.com/google/flatbuffers/go"
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
type RefMapEdit struct {
Name string
Addr hash.Hash
}
func RefMapLookup(rm *serial.RefMap, key string) hash.Hash {
var res hash.Hash
n := sort.Search(rm.NamesLength(), func(i int) bool {
return string(rm.Names(i)) >= key
})
if n != rm.NamesLength() && string(rm.Names(n)) == key {
copy(res[:], rm.RefArrayBytes()[n*20:])
}
return res
}
func RefMapApplyEdits(rm *serial.RefMap, builder *flatbuffers.Builder, edits []RefMapEdit) flatbuffers.UOffsetT {
sort.Slice(edits, func(i, j int) bool {
return edits[i].Name < edits[j].Name
})
type idx struct {
l, r int
}
var indexes []idx
ni := 0
ei := 0
for ni < rm.NamesLength() && ei < len(edits) {
if string(rm.Names(ni)) < edits[ei].Name {
indexes = append(indexes, idx{ni, -1})
ni += 1
} else if string(rm.Names(ni)) == edits[ei].Name {
if !edits[ei].Addr.IsEmpty() {
indexes = append(indexes, idx{-1, ei})
}
ei += 1
ni += 1
} else {
if !edits[ei].Addr.IsEmpty() {
indexes = append(indexes, idx{-1, ei})
}
ei += 1
}
}
for ni < rm.NamesLength() {
indexes = append(indexes, idx{ni, -1})
ni += 1
}
for ei < len(edits) {
if !edits[ei].Addr.IsEmpty() {
indexes = append(indexes, idx{-1, ei})
}
ei += 1
}
if len(indexes) == 0 {
serial.RefMapStart(builder)
return serial.RefMapEnd(builder)
}
nameoffs := make([]flatbuffers.UOffsetT, len(indexes))
for i := len(nameoffs) - 1; i >= 0; i-- {
var name string
if indexes[i].l != -1 {
name = string(rm.Names(indexes[i].l))
} else {
name = edits[indexes[i].r].Name
}
nameoffs[i] = builder.CreateString(name)
}
serial.RefMapStartNamesVector(builder, len(nameoffs))
for i := len(nameoffs) - 1; i >= 0; i-- {
builder.PrependUOffsetT(nameoffs[i])
}
namesoff := builder.EndVector(len(nameoffs))
hashsz := 20
hashessz := len(indexes) * hashsz
builder.Prep(flatbuffers.SizeUOffsetT, hashessz)
stop := int(builder.Head())
rmaddrbytes := rm.RefArrayBytes()
start := stop - hashessz
for _, idx := range indexes {
if idx.l != -1 {
copy(builder.Bytes[start:stop], rmaddrbytes[idx.l*20:idx.l*20+20])
} else {
copy(builder.Bytes[start:stop], edits[idx.r].Addr[:])
}
start += hashsz
}
start = stop - hashessz
refarrayoff := builder.CreateByteVector(builder.Bytes[start:stop])
serial.RefMapStart(builder)
serial.RefMapAddNames(builder, namesoff)
serial.RefMapAddRefArray(builder, refarrayoff)
serial.RefMapAddTreeCount(builder, uint64(len(indexes)))
serial.RefMapAddTreeLevel(builder, 0)
return serial.RefMapEnd(builder)
}
type refmap struct {
*serial.RefMap
}
func empty_refmap() refmap {
builder := flatbuffers.NewBuilder(24)
serial.RefMapStart(builder)
builder.Finish(serial.RefMapEnd(builder))
return refmap{serial.GetRootAsRefMap(builder.FinishedBytes(), 0)}
}
func (rm refmap) len() uint64 {
return uint64(rm.RefMap.NamesLength())
}
func (rm refmap) edit(edits []RefMapEdit) refmap {
func storeroot_flatbuffer(am prolly.AddressMap) []byte {
builder := flatbuffers.NewBuilder(1024)
builder.Finish(RefMapApplyEdits(rm.RefMap, builder, edits))
return refmap{serial.GetRootAsRefMap(builder.FinishedBytes(), 0)}
}
func (rm refmap) lookup(key string) hash.Hash {
return RefMapLookup(rm.RefMap, key)
}
func (rm refmap) set(key string, addr hash.Hash) refmap {
return rm.edit([]RefMapEdit{{key, addr}})
}
func (rm *refmap) delete(key string) refmap {
return rm.edit([]RefMapEdit{{key, hash.Hash{}}})
}
func (rm refmap) storeroot_flatbuffer() []byte {
builder := flatbuffers.NewBuilder(1024)
refmap := RefMapApplyEdits(rm.RefMap, builder, []RefMapEdit{})
ambytes := []byte(tree.ValueFromNode(am.Node()).(types.TupleRowStorage))
voff := builder.CreateByteVector(ambytes)
serial.StoreRootStart(builder)
serial.StoreRootAddRefs(builder, refmap)
serial.StoreRootAddAddressMap(builder, voff)
builder.FinishWithFileIdentifier(serial.StoreRootEnd(builder), []byte(serial.StoreRootFileID))
return builder.FinishedBytes()
}
func parse_storeroot(bs []byte) refmap {
func parse_storeroot(bs []byte, cs chunks.ChunkStore) prolly.AddressMap {
if !bytes.Equal([]byte(serial.StoreRootFileID), bs[4:8]) {
panic("expected store root file id, got: " + string(bs[4:8]))
}
sr := serial.GetRootAsStoreRoot(bs, 0)
rm := sr.Refs(nil)
if rm == nil {
panic("refmap of storeroot was missing")
}
if rm.TreeLevel() != 0 {
panic("unsupported multi-level refmap")
}
if uint64(rm.NamesLength()) != rm.TreeCount() {
panic("inconsistent refmap at level 0 where names length != tree count")
}
return refmap{rm}
mapbytes := sr.AddressMapBytes()
node := tree.NodeFromBytes(mapbytes)
return prolly.NewAddressMap(node, tree.NewNodeStore(cs))
}
go/store/datas/refmap_test.go
-81
View File
@@ -1,81 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datas
import (
"testing"
flatbuffers "github.com/google/flatbuffers/go"
"github.com/stretchr/testify/assert"
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/hash"
)
func TestRefMapLookupEmpty(t *testing.T) {
rm := empty_refmap().RefMap
assert.Equal(t, rm.NamesLength(), 0)
assert.Equal(t, RefMapLookup(rm, ""), hash.Hash{})
assert.Equal(t, RefMapLookup(rm, "doesnotexist"), hash.Hash{})
}
func edit_refmap(rm *serial.RefMap, edits []RefMapEdit) *serial.RefMap {
builder := flatbuffers.NewBuilder(1024)
builder.Finish(RefMapApplyEdits(rm, builder, edits))
return serial.GetRootAsRefMap(builder.FinishedBytes(), 0)
}
func TestRefMapEditInserts(t *testing.T) {
empty := empty_refmap().RefMap
a_hash := hash.Parse("3i50gcjrl9m2pgolrkc22kq46sj4p96o")
with_a := edit_refmap(empty, []RefMapEdit{RefMapEdit{"a", a_hash}})
assert.Equal(t, RefMapLookup(with_a, "a"), a_hash)
assert.Equal(t, RefMapLookup(with_a, "A"), hash.Hash{})
assert.Equal(t, RefMapLookup(empty, "a"), hash.Hash{})
b_hash := hash.Parse("7mm15d7prjlurr8g4u51n7dfg6bemt7p")
with_ab_from_a := edit_refmap(with_a, []RefMapEdit{RefMapEdit{"b", b_hash}})
with_ab_from_empty := edit_refmap(empty, []RefMapEdit{RefMapEdit{"b", b_hash}, RefMapEdit{"a", a_hash}})
assert.Equal(t, with_ab_from_a.Table().Bytes, with_ab_from_empty.Table().Bytes)
assert.Equal(t, RefMapLookup(with_ab_from_a, "a"), a_hash)
assert.Equal(t, RefMapLookup(with_ab_from_a, "b"), b_hash)
assert.Equal(t, RefMapLookup(with_ab_from_a, "c"), hash.Hash{})
assert.Equal(t, RefMapLookup(with_ab_from_a, "A"), hash.Hash{})
}
func TestRefMapEditDeletes(t *testing.T) {
empty := empty_refmap().RefMap
a_hash := hash.Parse("3i50gcjrl9m2pgolrkc22kq46sj4p96o")
b_hash := hash.Parse("7mm15d7prjlurr8g4u51n7dfg6bemt7p")
with_ab := edit_refmap(empty, []RefMapEdit{RefMapEdit{"b", b_hash}, RefMapEdit{"a", a_hash}})
without_a := edit_refmap(with_ab, []RefMapEdit{{Name: "a"}})
assert.Equal(t, RefMapLookup(without_a, "a"), hash.Hash{})
assert.Equal(t, RefMapLookup(without_a, "b"), b_hash)
without_ab := edit_refmap(without_a, []RefMapEdit{{Name: "b"}})
assert.Equal(t, without_ab.NamesLength(), 0)
assert.Equal(t, without_ab.RefArrayLength(), 0)
assert.Equal(t, RefMapLookup(without_ab, "a"), hash.Hash{})
assert.Equal(t, RefMapLookup(without_ab, "b"), hash.Hash{})
assert.Equal(t, empty.Table().Bytes, without_ab.Table().Bytes)
with_b := edit_refmap(empty, []RefMapEdit{RefMapEdit{"b", b_hash}})
assert.Equal(t, without_a.Table().Bytes, with_b.Table().Bytes)
delete_from_empty := edit_refmap(empty, []RefMapEdit{RefMapEdit{Name: "b"}})
assert.Equal(t, delete_from_empty.Table().Bytes, empty.Table().Bytes)
}
go/store/prolly/address_map.go
+9 -1
View File
@@ -30,9 +30,13 @@ type AddressMap struct {
}
func NewEmptyAddressMap(ns tree.NodeStore) AddressMap {
return NewAddressMap(newEmptyMapNode(ns.Pool()), ns)
}
func NewAddressMap(node tree.Node, ns tree.NodeStore) AddressMap {
return AddressMap{
addresses: orderedTree[stringSlice, address, lexicographic]{
root: newEmptyMapNode(ns.Pool()),
root: node,
ns: ns,
order: lexicographic{},
},
@@ -59,6 +63,10 @@ func (c AddressMap) Height() int {
return c.addresses.height()
}
func (c AddressMap) Node() tree.Node {
return c.addresses.root
}
func (c AddressMap) HashOf() hash.Hash {
return c.addresses.hashOf()
}
go/store/prolly/conflicts_map.go
+4
View File
@@ -84,6 +84,10 @@ func (c ConflictMap) HashOf() hash.Hash {
return c.conflicts.hashOf()
}
func (c ConflictMap) Node() tree.Node {
return c.conflicts.root
}
func (c ConflictMap) Format() *types.NomsBinFormat {
return c.conflicts.ns.Format()
}
go/store/prolly/map.go
+20
View File
@@ -166,6 +166,10 @@ func (m Map) IterRange(ctx context.Context, rng Range) (MapIter, error) {
}
}
func (m Map) Node() tree.Node {
return m.tuples.root
}
// Pool returns the pool.BuffPool of the underlying tuples' tree.NodeStore
func (m Map) Pool() pool.BuffPool {
return m.tuples.ns.Pool()
@@ -286,3 +290,19 @@ func DebugFormat(ctx context.Context, m Map) (string, error) {
sb.WriteString("}")
return sb.String(), nil
}
// ConvertToKeylessIndex converts the given map to a keyless index map.
func ConvertToKeylessIndex(m Map) Map {
keyDesc, valDesc := m.Descriptors()
newTypes := make([]val.Type, len(keyDesc.Types)+1)
copy(newTypes, keyDesc.Types)
newTypes[len(newTypes)-1] = val.Type{Enc: val.Hash128Enc}
newKeyDesc := val.NewTupleDescriptorWithComparator(keyDesc.Comparator(), newTypes...)
newTuples := m.tuples
newTuples.order = newKeyDesc
return Map{
tuples: newTuples,
keyDesc: newKeyDesc,
valDesc: valDesc,
}
}
go/store/prolly/map_test.go
+1 -1
View File
@@ -95,7 +95,7 @@ func TestNewEmptyNode(t *testing.T) {
assert.Equal(t, 0, empty.Level())
assert.Equal(t, 0, empty.Count())
assert.Equal(t, 0, empty.TreeCount())
assert.Equal(t, 72, empty.Size())
assert.Equal(t, 75, empty.Size())
assert.True(t, empty.IsLeaf())
}
go/store/prolly/message/address_map.go
+31 -9
View File
@@ -74,26 +74,47 @@ func (s AddressMapSerializer) Serialize(keys, addrs [][]byte, subtrees []uint64,
serial.AddressMapAddTreeCount(b, uint64(len(keys)))
}
serial.AddressMapAddTreeLevel(b, uint8(level))
b.FinishWithFileIdentifier(serial.AddressMapEnd(b), addressMapFileID)
return b.FinishedBytes()
return finishMessage(b, serial.AddressMapEnd(b), addressMapFileID)
}
func finishMessage(b *fb.Builder, off fb.UOffsetT, fileID []byte) []byte {
// We finish the buffer by prefixing it with:
// 1) 1 byte NomsKind == TupleRowStorage.
// 2) big endian uint16 representing the size of the message, not
// including the kind or size prefix bytes.
//
// This allows chunks we serialize here to be read by types binary
// codec.
//
// All accessors in this package expect this prefix to be on the front
// of the message bytes as well. See |messagePrefixSz|.
b.Prep(1, fb.SizeInt32+4+messagePrefixSz)
b.FinishWithFileIdentifier(off, fileID)
bytes := b.Bytes[b.Head()-messagePrefixSz:]
bytes[0] = byte(MessageTypesKind)
binary.BigEndian.PutUint16(bytes[1:], uint16(len(b.Bytes)-int(b.Head())))
return bytes
}
func getAddressMapKeys(msg Message) (keys val.SlicedBuffer) {
am := serial.GetRootAsAddressMap(msg, 0)
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
keys.Buf = am.KeyItemsBytes()
keys.Offs = getAddressMapKeyOffsets(am)
return
}
func getAddressMapValues(msg Message) (values val.SlicedBuffer) {
am := serial.GetRootAsAddressMap(msg, 0)
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
values.Buf = am.AddressArrayBytes()
values.Offs = offsetsForAddressArray(values.Buf)
return
}
func walkAddressMapAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error {
am := serial.GetRootAsAddressMap(msg, 0)
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
arr := am.AddressArrayBytes()
for i := 0; i < len(arr)/hash.ByteLen; i++ {
addr := hash.New(arr[i*addrSize : (i+1)*addrSize])
@@ -105,7 +126,7 @@ func walkAddressMapAddresses(ctx context.Context, msg Message, cb func(ctx conte
}
func getAddressMapCount(msg Message) uint16 {
am := serial.GetRootAsAddressMap(msg, 0)
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
if am.KeyItemsLength() == 0 {
return 0
}
@@ -114,18 +135,18 @@ func getAddressMapCount(msg Message) uint16 {
}
func getAddressMapTreeLevel(msg Message) int {
am := serial.GetRootAsAddressMap(msg, 0)
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
return int(am.TreeLevel())
}
func getAddressMapTreeCount(msg Message) int {
am := serial.GetRootAsAddressMap(msg, 0)
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
return int(am.TreeCount())
}
func getAddressMapSubtrees(msg Message) []uint64 {
counts := make([]uint64, getAddressMapCount(msg))
am := serial.GetRootAsAddressMap(msg, 0)
am := serial.GetRootAsAddressMap(msg, messagePrefixSz)
return decodeVarints(am.SubtreeCountsBytes(), counts)
}
@@ -149,5 +170,6 @@ func estimateAddressMapSize(keys, addresses [][]byte, subtrees []uint64) (keySz,
totalSz += len(subtrees) * binary.MaxVarintLen64
totalSz += 8 + 1 + 1 + 1
totalSz += 72
totalSz += messagePrefixSz
return
}
go/store/prolly/message/message.go
+9 -5
View File
@@ -23,6 +23,10 @@ import (
"github.com/dolthub/dolt/go/store/val"
)
const MessageTypesKind int = 28
const messagePrefixSz = 3
type Message []byte
type Serializer interface {
@@ -30,7 +34,7 @@ type Serializer interface {
}
func GetKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt uint16) {
id := serial.GetFileID(msg)
id := serial.GetFileID(msg[messagePrefixSz:])
if id == serial.ProllyTreeNodeFileID {
return getProllyMapKeysAndValues(msg)
@@ -46,7 +50,7 @@ func GetKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt uint16) {
}
func WalkAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error {
id := serial.GetFileID(msg)
id := serial.GetFileID(msg[messagePrefixSz:])
switch id {
case serial.ProllyTreeNodeFileID:
return walkProllyMapAddresses(ctx, msg, cb)
@@ -58,7 +62,7 @@ func WalkAddresses(ctx context.Context, msg Message, cb func(ctx context.Context
}
func GetTreeLevel(msg Message) int {
id := serial.GetFileID(msg)
id := serial.GetFileID(msg[messagePrefixSz:])
switch id {
case serial.ProllyTreeNodeFileID:
return getProllyMapTreeLevel(msg)
@@ -70,7 +74,7 @@ func GetTreeLevel(msg Message) int {
}
func GetTreeCount(msg Message) int {
id := serial.GetFileID(msg)
id := serial.GetFileID(msg[messagePrefixSz:])
switch id {
case serial.ProllyTreeNodeFileID:
return getProllyMapTreeCount(msg)
@@ -82,7 +86,7 @@ func GetTreeCount(msg Message) int {
}
func GetSubtrees(msg Message) []uint64 {
id := serial.GetFileID(msg)
id := serial.GetFileID(msg[messagePrefixSz:])
switch id {
case serial.ProllyTreeNodeFileID:
return getProllyMapSubtrees(msg)
go/store/prolly/message/prolly_map.go
+9 -8
View File
@@ -85,12 +85,12 @@ func (s ProllyMapSerializer) Serialize(keys, values [][]byte, subtrees []uint64,
serial.ProllyTreeNodeAddKeyType(b, serial.ItemTypeTupleFormatAlpha)
serial.ProllyTreeNodeAddValueType(b, serial.ItemTypeTupleFormatAlpha)
serial.ProllyTreeNodeAddTreeLevel(b, uint8(level))
b.FinishWithFileIdentifier(serial.ProllyTreeNodeEnd(b), prollyMapFileID)
return b.FinishedBytes()
return finishMessage(b, serial.ProllyTreeNodeEnd(b), prollyMapFileID)
}
func getProllyMapKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt uint16) {
pm := serial.GetRootAsProllyTreeNode(msg, 0)
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
keys.Buf = pm.KeyItemsBytes()
keys.Offs = getProllyMapKeyOffsets(pm)
@@ -113,7 +113,7 @@ func getProllyMapKeysAndValues(msg Message) (keys, values val.SlicedBuffer, cnt
}
func walkProllyMapAddresses(ctx context.Context, msg Message, cb func(ctx context.Context, addr hash.Hash) error) error {
pm := serial.GetRootAsProllyTreeNode(msg, 0)
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
arr := pm.AddressArrayBytes()
for i := 0; i < len(arr)/hash.ByteLen; i++ {
addr := hash.New(arr[i*addrSize : (i+1)*addrSize])
@@ -136,7 +136,7 @@ func walkProllyMapAddresses(ctx context.Context, msg Message, cb func(ctx contex
}
func getProllyMapCount(msg Message) uint16 {
pm := serial.GetRootAsProllyTreeNode(msg, 0)
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
if pm.KeyItemsLength() == 0 {
return 0
}
@@ -145,18 +145,18 @@ func getProllyMapCount(msg Message) uint16 {
}
func getProllyMapTreeLevel(msg Message) int {
pm := serial.GetRootAsProllyTreeNode(msg, 0)
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
return int(pm.TreeLevel())
}
func getProllyMapTreeCount(msg Message) int {
pm := serial.GetRootAsProllyTreeNode(msg, 0)
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
return int(pm.TreeCount())
}
func getProllyMapSubtrees(msg Message) []uint64 {
counts := make([]uint64, getProllyMapCount(msg))
pm := serial.GetRootAsProllyTreeNode(msg, 0)
pm := serial.GetRootAsProllyTreeNode(msg, messagePrefixSz)
return decodeVarints(pm.SubtreeCountsBytes(), counts)
}
@@ -204,6 +204,7 @@ func estimateProllyMapSize(keys, values [][]byte, subtrees []uint64) (keySz, val
bufSz += 8 + 1 + 1 + 1 // metadata
bufSz += 72 // vtable (approx)
bufSz += 100 // padding?
bufSz += messagePrefixSz
return
}
go/store/prolly/shim.go → go/store/prolly/shim/shim.go
+10 -24
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package prolly
package shim
import (
"fmt"
@@ -22,6 +22,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
@@ -31,29 +32,29 @@ func NodeFromValue(v types.Value) tree.Node {
return tree.NodeFromBytes(v.(types.TupleRowStorage))
}
func ValueFromMap(m Map) types.Value {
return tree.ValueFromNode(m.tuples.root)
func ValueFromMap(m prolly.Map) types.Value {
return tree.ValueFromNode(m.Node())
}
func ValueFromConflictMap(m ConflictMap) types.Value {
return tree.ValueFromNode(m.conflicts.root)
func ValueFromConflictMap(m prolly.ConflictMap) types.Value {
return tree.ValueFromNode(m.Node())
}
func MapFromValue(v types.Value, sch schema.Schema, vrw types.ValueReadWriter) Map {
func MapFromValue(v types.Value, sch schema.Schema, vrw types.ValueReadWriter) prolly.Map {
root := NodeFromValue(v)
kd := KeyDescriptorFromSchema(sch)
vd := ValueDescriptorFromSchema(sch)
ns := tree.NewNodeStore(ChunkStoreFromVRW(vrw))
return NewMap(root, ns, kd, vd)
return prolly.NewMap(root, ns, kd, vd)
}
func ConflictMapFromValue(v types.Value, ourSchema, theirSchema, baseSchema schema.Schema, vrw types.ValueReadWriter) ConflictMap {
func ConflictMapFromValue(v types.Value, ourSchema, theirSchema, baseSchema schema.Schema, vrw types.ValueReadWriter) prolly.ConflictMap {
root := NodeFromValue(v)
kd, ourVD := MapDescriptorsFromSchema(ourSchema)
theirVD := ValueDescriptorFromSchema(theirSchema)
baseVD := ValueDescriptorFromSchema(baseSchema)
ns := tree.NewNodeStore(ChunkStoreFromVRW(vrw))
return NewConflictMap(root, ns, kd, ourVD, theirVD, baseVD)
return prolly.NewConflictMap(root, ns, kd, ourVD, theirVD, baseVD)
}
func ChunkStoreFromVRW(vrw types.ValueReadWriter) chunks.ChunkStore {
@@ -113,21 +114,6 @@ func ValueDescriptorFromSchema(sch schema.Schema) val.TupleDesc {
return val.NewTupleDescriptor(tt...)
}
// ConvertToKeylessIndex converts the given map to a keyless index map.
func ConvertToKeylessIndex(m Map) Map {
newTypes := make([]val.Type, len(m.keyDesc.Types)+1)
copy(newTypes, m.keyDesc.Types)
newTypes[len(newTypes)-1] = val.Type{Enc: val.Hash128Enc}
newKeyDesc := val.NewTupleDescriptorWithComparator(m.keyDesc.Comparator(), newTypes...)
newTuples := m.tuples
newTuples.order = newKeyDesc
return Map{
tuples: newTuples,
keyDesc: newKeyDesc,
valDesc: m.valDesc,
}
}
func encodingFromSqlType(typ query.Type) val.Encoding {
// todo(andy): replace temp encodings
go/store/prolly/tree/content_address_test.go
+5 -4
View File
@@ -27,10 +27,11 @@ import (
)
var goldenHash = hash.Hash{
0xcb, 0x1d, 0xe1, 0xbd, 0x4a,
0x3c, 0x27, 0x18, 0xe, 0xba,
0xb7, 0xe8, 0x30, 0x44, 0x69,
0x3d, 0x86, 0x27, 0x26, 0x2d,
0x39, 0x1c, 0xcb, 0xd8,
0xea, 0xd2, 0xdc, 0x42,
0x76, 0xb7, 0x38, 0xf1,
0x0d, 0x4f, 0x48, 0x91,
0x7d, 0x1f, 0xb7, 0xb4,
}
// todo(andy): need and analogous test in pkg prolly
go/store/prolly/tree/immutable_tree.go
+150
View File
@@ -0,0 +1,150 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tree
import (
"context"
"errors"
"io"
"github.com/dolthub/dolt/go/store/prolly/message"
)
const defaultFixedChunkLength = 4000
var ErrInvalidChunkSize = errors.New("invalid chunkSize; value must be > 1")
// buildImmutableTree writes the contents of |reader| as an append-only
// tree, returning the root node or an error if applicable. |chunkSize|
// fixes the split size of leaf and intermediate node chunks.
func buildImmutableTree(ctx context.Context, r io.Reader, ns NodeStore, S message.Serializer, chunkSize int) (Node, error) {
if chunkSize <= 1 {
return Node{}, ErrInvalidChunkSize
}
var levels [][]novelNode
var levelCnts []int
var finalize bool
// We use lookahead to check whether the reader has
// more bytes. The reader will only EOF when reading
// zero bytes into the lookahead buffer, but we want
// to know at the beginning of a loop whether we are
// finished.
lookahead := make([]byte, chunkSize)
lookaheadN, err := r.Read(lookahead)
if err != nil {
return Node{}, err
}
buf := make([]byte, chunkSize)
for {
copy(buf, lookahead)
curN := lookaheadN
lookaheadN, err = r.Read(lookahead)
if err == io.EOF {
finalize = true
} else if err != nil {
return Node{}, err
}
novel, err := _newLeaf(ctx, ns, S, buf[:curN])
if err != nil {
return Node{}, err
}
i := 0
for {
// Three cases for building tree
// 1) reached new level => create new level
// 2) add novel node to current level
// 3) we didn't fill the current level => break
// 4) we filled current level, chunk and recurse into parent
//
// Two cases for finalizing tree
// 1) we haven't hit root, so we add the final chunk, finalize level, and continue upwards
// 2) we overshot root finalizing chunks, and we return the single root in the lower level
if i > len(levels)-1 {
levels = append(levels, make([]novelNode, chunkSize))
levelCnts = append(levelCnts, 0)
}
levels[i][levelCnts[i]] = novel
levelCnts[i]++
if levelCnts[i] < chunkSize {
// current level is not full
if !finalize {
// only continue and chunk this level if finalizing all in-progress nodes
break
}
}
nodes := levels[i][:levelCnts[i]]
if len(nodes) == 1 && i == len(levels)-1 {
// this is necessary and only possible if we're finalizing
// note: this is the only non-error return
return nodes[0].node, nil
}
// chunk the current level
novel, err = _newInternal(ctx, ns, S, nodes, i+1, chunkSize)
if err != nil {
return Node{}, err
}
levelCnts[i] = 0
i++
}
}
}
func _newInternal(ctx context.Context, ns NodeStore, s message.Serializer, nodes []novelNode, level int, chunkSize int) (novelNode, error) {
keys := make([][]byte, len(nodes))
vals := make([][]byte, len(nodes))
subtrees := make([]uint64, len(nodes))
treeCnt := uint64(0)
for i := range nodes {
keys[i] = []byte{0}
vals[i] = nodes[i].addr[:]
subtrees[i] = nodes[i].treeCount
treeCnt += nodes[i].treeCount
}
msg := s.Serialize(keys, vals, subtrees, level)
node := NodeFromBytes(msg)
addr, err := ns.Write(ctx, node)
if err != nil {
return novelNode{}, err
}
return novelNode{
addr: addr,
node: node,
lastKey: []byte{0},
treeCount: treeCnt,
}, nil
}
func _newLeaf(ctx context.Context, ns NodeStore, s message.Serializer, buf []byte) (novelNode, error) {
msg := s.Serialize([][]byte{{0}}, [][]byte{buf}, []uint64{1}, 0)
node := NodeFromBytes(msg)
addr, err := ns.Write(ctx, node)
if err != nil {
return novelNode{}, err
}
return novelNode{
addr: addr,
node: node,
lastKey: []byte{0},
treeCount: 1,
}, nil
}
go/store/prolly/tree/immutable_tree_test.go
+198
View File
@@ -0,0 +1,198 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tree
import (
"bytes"
"context"
"errors"
"fmt"
"math"
"testing"
"github.com/stretchr/testify/require"
"github.com/dolthub/dolt/go/store/prolly/message"
)
func TestWriteImmutableTree(t *testing.T) {
tests := []struct {
inputSize int
chunkSize int
err error
}{
{
inputSize: 100,
chunkSize: 5,
},
{
inputSize: 100,
chunkSize: 100,
},
{
inputSize: 100,
chunkSize: 101,
},
{
inputSize: 255,
chunkSize: 5,
},
{
inputSize: 243,
chunkSize: 5,
},
{
inputSize: 47,
chunkSize: 3,
},
{
inputSize: 200,
chunkSize: 7,
},
{
inputSize: 200,
chunkSize: 40,
},
{
inputSize: 1,
chunkSize: 5,
},
{
inputSize: 20,
chunkSize: 500,
},
{
inputSize: 10,
chunkSize: 1,
err: ErrInvalidChunkSize,
},
{
inputSize: 10,
chunkSize: -1,
err: ErrInvalidChunkSize,
},
}
for _, tt := range tests {
t.Run(fmt.Sprintf("inputSize=%d; chunkSize=%d", tt.inputSize, tt.chunkSize), func(t *testing.T) {
buf := make([]byte, tt.inputSize)
for i := range buf {
buf[i] = byte(i)
}
ctx := context.Background()
r := bytes.NewReader(buf)
ns := NewTestNodeStore()
serializer := message.ProllyMapSerializer{Pool: ns.Pool()}
root, err := buildImmutableTree(ctx, r, ns, serializer, tt.chunkSize)
if tt.err != nil {
require.True(t, errors.Is(err, tt.err))
return
}
require.NoError(t, err)
expSubtrees := expectedSubtrees(tt.inputSize, tt.chunkSize)
expLevel := expectedLevel(tt.inputSize, tt.chunkSize)
expSum := expectedSum(tt.inputSize)
expUnfilled := expectedUnfilled(tt.inputSize, tt.chunkSize)
unfilledCnt := 0
sum := 0
byteCnt := 0
WalkNodes(ctx, root, ns, func(ctx context.Context, n Node) error {
var keyCnt int
if n.IsLeaf() {
byteCnt += len(n.values.Buf)
for _, i := range n.getValue(0) {
sum += int(i)
}
keyCnt = len(n.values.Buf)
} else {
keyCnt = n.Count()
}
if keyCnt != tt.chunkSize {
unfilledCnt += 1
}
return nil
})
require.Equal(t, expLevel, root.Level())
require.Equal(t, expSum, sum)
require.Equal(t, tt.inputSize, byteCnt)
require.Equal(t, expUnfilled, unfilledCnt)
require.Equal(t, expSubtrees, root.getSubtreeCounts())
})
}
}
func expectedLevel(size, chunk int) int {
l := 0
for size > chunk {
size = size / chunk
l += 1
}
return l
}
func expectedSubtrees(size, chunk int) SubtreeCounts {
if size <= chunk {
return SubtreeCounts{0}
}
size = int(math.Ceil(float64(size) / float64(chunk)))
l := chunk
for l < size {
l *= chunk
}
l /= chunk
res := make(SubtreeCounts, 0)
for size > l {
res = append(res, uint64(l))
size -= l
}
res = append(res, uint64(size))
return res
}
func expectedSum(size int) int {
return (size * (size + 1) / 2) - size
}
func expectedUnfilled(size, chunk int) int {
l := chunk
for l < size {
l *= chunk
}
l /= chunk
size -= l
cnt := 0
i := 1
for size > 0 {
if l > size {
if i < chunk-1 {
cnt += 1
}
l /= chunk
i = 0
} else {
size -= l
i++
}
}
if i < chunk {
cnt += 1
}
return cnt
}
go/store/prolly/tree/node.go
+17
View File
@@ -180,6 +180,23 @@ func OutputProllyNode(w io.Writer, node Node) error {
return nil
}
func OutputAddressMapNode(w io.Writer, node Node) error {
w.Write([]byte("["))
for i := 0; i < int(node.count); i++ {
k := node.GetKey(i)
w.Write([]byte("\n { key: "))
w.Write(k)
ref := node.getAddress(i)
w.Write([]byte(" ref: #"))
w.Write([]byte(ref.String()))
w.Write([]byte(" }"))
}
w.Write([]byte("\n]\n"))
return nil
}
func ValueFromNode(root Node) types.Value {
return types.TupleRowStorage(root.bytes())
}
go/store/prolly/tree/node_store.go
+5 -2
View File
@@ -99,6 +99,9 @@ func (ns nodeStore) Pool() pool.BuffPool {
}
func (ns nodeStore) Format() *types.NomsBinFormat {
// todo(andy): read from |ns.store|
return types.Format_DOLT_1
nbf, err := types.GetFormatForVersionString(ns.store.Version())
if err != nil {
panic(err)
}
return nbf
}
go/store/prolly/tree/node_test.go
+41
View File
@@ -15,6 +15,7 @@
package tree
import (
"context"
"math"
"math/rand"
"testing"
@@ -23,7 +24,10 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/message"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
@@ -66,6 +70,43 @@ func TestNodeSize(t *testing.T) {
assert.Equal(t, 128, int(sz))
}
func TestNodeHashValueCompatibility(t *testing.T) {
keys, values := randomNodeItemPairs(t, (rand.Int()%101)+50)
nd := newLeafNode(keys, values)
nbf := types.Format_DOLT_1
th, err := ValueFromNode(nd).Hash(nbf)
require.NoError(t, err)
assert.Equal(t, nd.HashOf(), th)
h1 := hash.Parse("kvup5vdur99ush7c18g0kjc6rhdkfdgo")
h2 := hash.Parse("7e54ill10nji9oao1ja88buh9itaj7k9")
msg := message.AddressMapSerializer{Pool: sharedPool}.Serialize(
[][]byte{[]byte("chopin"), []byte("listz")},
[][]byte{h1[:], h2[:]},
[]uint64{},
0)
nd = NodeFromBytes(msg)
th, err = ValueFromNode(nd).Hash(nbf)
require.NoError(t, err)
assert.Equal(t, nd.HashOf(), th)
}
func TestNodeDecodeValueCompatibility(t *testing.T) {
keys, values := randomNodeItemPairs(t, (rand.Int()%101)+50)
nd := newLeafNode(keys, values)
ts := &chunks.TestStorage{}
cs := ts.NewView()
ns := NewNodeStore(cs)
vs := types.NewValueStore(cs)
h, err := ns.Write(context.Background(), nd)
require.NoError(t, err)
v, err := vs.ReadValue(context.Background(), h)
require.NoError(t, err)
assert.Equal(t, nd.bytes(), []byte(v.(types.TupleRowStorage)))
}
func randomNodeItemPairs(t *testing.T, count int) (keys, values []Item) {
keys = make([]Item, count)
for i := range keys {
go/store/types/codec.go
+8
View File
@@ -325,6 +325,14 @@ func (b *binaryNomsReader) ReadInlineBlob() []byte {
return bytes
}
func (b *binaryNomsReader) readTupleRowStorage() []byte {
size := uint32(b.readUint16())
// start at offset-3, to include the kind byte + Uint16 for size...
bytes := b.buff[b.offset-3 : b.offset+size]
b.offset += size
return bytes
}
func (b *binaryNomsReader) ReadUUID() uuid.UUID {
id := uuid.UUID{}
copy(id[:uuidNumBytes], b.readBytes(uuidNumBytes))
go/store/types/noms_kind.go
+8
View File
@@ -21,6 +21,10 @@
package types
import (
"github.com/dolthub/dolt/go/store/prolly/message"
)
// NomsKind allows a TypeDesc to indicate what kind of type is described.
type NomsKind uint8
@@ -125,6 +129,10 @@ func init() {
SupportedKinds[PolygonKind] = true
SupportedKinds[SerialMessageKind] = true
SupportedKinds[TupleRowStorageKind] = true
if message.MessageTypesKind != int(TupleRowStorageKind) {
panic("internal error: message.MessageTypesKind != TupleRowStorageKind")
}
}
var KindToTypeSlice []Value
go/store/types/serial_message.go
+13 -61
View File
@@ -56,15 +56,8 @@ func (sm SerialMessage) HumanReadableString() string {
case serial.StoreRootFileID:
msg := serial.GetRootAsStoreRoot([]byte(sm), 0)
ret := &strings.Builder{}
refs := msg.Refs(nil)
fmt.Fprintf(ret, "{\n")
hashes := refs.RefArrayBytes()
for i := 0; i < refs.NamesLength(); i++ {
name := refs.Names(i)
addr := hash.New(hashes[i*20 : (i+1)*20])
fmt.Fprintf(ret, "\t%s: #%s\n", name, addr.String())
}
fmt.Fprintf(ret, "}")
mapbytes := msg.AddressMapBytes()
fmt.Fprintf(ret, "StoreRoot{%s}", TupleRowStorage(mapbytes).HumanReadableString())
return ret.String()
case serial.TagFileID:
return "Tag"
@@ -115,14 +108,7 @@ func (sm SerialMessage) HumanReadableString() string {
fmt.Fprintf(ret, "\tFeatureVersion: %d\n", msg.FeatureVersion())
fmt.Fprintf(ret, "\tForeignKeys: #%s\n", hash.New(msg.ForeignKeyAddrBytes()).String())
fmt.Fprintf(ret, "\tSuperSchema: #%s\n", hash.New(msg.SuperSchemasAddrBytes()).String())
fmt.Fprintf(ret, "\tTables: {\n")
tableRefs := msg.Tables(nil)
hashes := tableRefs.RefArrayBytes()
for i := 0; i < tableRefs.NamesLength(); i++ {
name := tableRefs.Names(i)
addr := hash.New(hashes[i*20 : (i+1)*20])
fmt.Fprintf(ret, "\t\t%s: #%s\n", name, addr.String())
}
fmt.Fprintf(ret, "\tTables: {\n\t%s", TupleRowStorage(msg.TablesBytes()).HumanReadableString())
fmt.Fprintf(ret, "\t}\n")
fmt.Fprintf(ret, "}")
return ret.String()
@@ -140,14 +126,7 @@ func (sm SerialMessage) HumanReadableString() string {
// TODO: can't use tree package to print here, creates a cycle
fmt.Fprintf(ret, "\tPrimary index: prolly tree\n")
fmt.Fprintf(ret, "\tSecondary indexes: {\n")
idxRefs := msg.SecondaryIndexes(nil)
hashes := idxRefs.RefArrayBytes()
for i := 0; i < idxRefs.NamesLength(); i++ {
name := idxRefs.Names(i)
addr := hash.New(hashes[i*20 : (i+1)*20])
fmt.Fprintf(ret, "\t\t%s: #%s\n", name, addr.String())
}
fmt.Fprintf(ret, "\tSecondary indexes: {\n\t%s\n", TupleRowStorage(msg.SecondaryIndexesBytes()).HumanReadableString())
fmt.Fprintf(ret, "\t}\n")
fmt.Fprintf(ret, "}")
return ret.String()
@@ -175,18 +154,9 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
switch serial.GetFileID([]byte(sm)) {
case serial.StoreRootFileID:
msg := serial.GetRootAsStoreRoot([]byte(sm), 0)
rm := msg.Refs(nil)
refs := rm.RefArrayBytes()
for i := 0; i < rm.NamesLength(); i++ {
off := i * 20
addr := hash.New(refs[off : off+20])
r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight)
if err != nil {
return err
}
if err = cb(r); err != nil {
return err
}
if msg.AddressMapLength() > 0 {
mapbytes := msg.AddressMapBytes()
return TupleRowStorage(mapbytes).walkRefs(nbf, cb)
}
case serial.TagFileID:
msg := serial.GetRootAsTag([]byte(sm), 0)
@@ -238,18 +208,9 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
}
case serial.RootValueFileID:
msg := serial.GetRootAsRootValue([]byte(sm), 0)
rm := msg.Tables(nil)
refs := rm.RefArrayBytes()
for i := 0; i < rm.NamesLength(); i++ {
off := i * 20
addr := hash.New(refs[off : off+20])
r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight)
if err != nil {
return err
}
if err = cb(r); err != nil {
return err
}
err := TupleRowStorage(msg.TablesBytes()).walkRefs(nbf, cb)
if err != nil {
return err
}
addr := hash.New(msg.ForeignKeyAddrBytes())
if !addr.IsEmpty() {
@@ -339,18 +300,9 @@ func (sm SerialMessage) walkRefs(nbf *NomsBinFormat, cb RefCallback) error {
}
}
rm := msg.SecondaryIndexes(nil)
refs := rm.RefArrayBytes()
for i := 0; i < rm.NamesLength(); i++ {
off := i * 20
addr := hash.New(refs[off : off+20])
r, err := constructRef(nbf, addr, PrimitiveTypeMap[ValueKind], SerialMessageRefHeight)
if err != nil {
return err
}
if err = cb(r); err != nil {
return err
}
err = TupleRowStorage(msg.SecondaryIndexesBytes()).walkRefs(nbf, cb)
if err != nil {
return err
}
mapbytes := msg.PrimaryIndexBytes()
go/store/types/tuplerowstorage.go
+28 -17
View File
@@ -18,16 +18,21 @@ import (
"bytes"
"context"
"encoding/hex"
"fmt"
"math"
"strings"
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/message"
)
// TupleRowStorage is a clone of InlineBlob. It only exists to be able to easily differentiate these two very different
// use cases during the migration from the old storage format to the new one.
// TupleRowStorage cribs its implementation from InlineBlob. It bridges
// prolly/message byte arrays between types.Value and prolly/tree.Node.
//
// Unlike SerialMessage, the byte array held in TupleRowStorage includes the
// NomsKind byte and the BigEndian uint16 size of the message. |writeTo| is
// simply a call through to writeRaw, and |readFrom| has to pick up bytes from
// the reader that have already been "read" to determine kind and size.
type TupleRowStorage []byte
func (v TupleRowStorage) Value(ctx context.Context) (Value, error) {
@@ -81,23 +86,12 @@ func (v TupleRowStorage) valueReadWriter() ValueReadWriter {
}
func (v TupleRowStorage) writeTo(w nomsWriter, nbf *NomsBinFormat) error {
byteLen := len(v)
if byteLen > math.MaxUint16 {
return fmt.Errorf("TupleRowStorage has length %v when max is %v", byteLen, math.MaxUint16)
}
err := TupleRowStorageKind.writeTo(w, nbf)
if err != nil {
return err
}
w.writeUint16(uint16(byteLen))
w.writeRaw(v)
return nil
}
func (v TupleRowStorage) readFrom(nbf *NomsBinFormat, b *binaryNomsReader) (Value, error) {
bytes := b.ReadInlineBlob()
bytes := b.readTupleRowStorage()
return TupleRowStorage(bytes), nil
}
@@ -107,5 +101,22 @@ func (v TupleRowStorage) skip(nbf *NomsBinFormat, b *binaryNomsReader) {
}
func (v TupleRowStorage) HumanReadableString() string {
return strings.ToUpper(hex.EncodeToString(v))
if serial.GetFileID(v) == serial.AddressMapFileID {
keys, values, cnt := message.GetKeysAndValues(message.Message([]byte(v)))
var b strings.Builder
b.Write([]byte("AddressMap{\n"))
for i := uint16(0); i < cnt; i++ {
name := keys.GetSlice(int(i))
addr := values.GetSlice(int(i))
b.Write([]byte("\t"))
b.Write(name)
b.Write([]byte(": "))
b.Write([]byte(hash.New(addr).String()))
b.Write([]byte("\n"))
}
b.Write([]byte("}"))
return b.String()
} else {
return strings.ToUpper(hex.EncodeToString(v))
}
}
integration-tests/bats/auto_increment.bats
+4 -1
View File
@@ -3,7 +3,6 @@ load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
setup_common
skip_nbf_dolt_1
dolt sql <<SQL
CREATE TABLE test (
@@ -435,6 +434,8 @@ SQL
}
@test "auto_increment: dolt_merge() works with no auto increment overlap" {
skip_nbf_dolt_1
dolt sql <<SQL
CREATE TABLE t (
pk int PRIMARY KEY AUTO_INCREMENT,
@@ -499,6 +500,8 @@ SQL
}
@test "auto_increment: dolt_merge() with a gap in an auto increment key" {
skip_nbf_dolt_1
dolt sql <<SQL
CREATE TABLE t (
pk int PRIMARY KEY AUTO_INCREMENT,
integration-tests/bats/branch.bats
-1
View File
@@ -11,7 +11,6 @@ teardown() {
}
@test "branch: deleting a branch deletes its working set" {
skip_nbf_dolt_1
dolt checkout -b to_delete
root=$(noms root .dolt/noms)
integration-tests/bats/column_tags.bats
-5
View File
@@ -11,7 +11,6 @@ teardown() {
}
@test "column_tags: Renaming a column should preserve the tag number" {
skip_nbf_dolt_1
dolt sql <<SQL
CREATE TABLE test (
pk BIGINT NOT NULL,
@@ -60,7 +59,6 @@ SQL
@test "column_tags: Merging two branches that added same tag, name, type, and constraints" {
skip_nbf_dolt_1
dolt sql <<SQL
CREATE TABLE test (
pk BIGINT NOT NULL,
@@ -85,7 +83,6 @@ SQL
}
@test "column_tags: Merging branches that use the same tag referring to different schema fails" {
skip_nbf_dolt_1
dolt sql <<SQL
CREATE TABLE test (
pk BIGINT NOT NULL COMMENT 'tag:1234',
@@ -111,7 +108,6 @@ SQL
}
@test "column_tags: Merging branches that use the same tag referring to different column names fails" {
skip_nbf_dolt_1
dolt sql <<SQL
CREATE TABLE test (
pk BIGINT NOT NULL COMMENT 'tag:1234',
@@ -138,7 +134,6 @@ SQL
}
@test "column_tags: Merging branches that both created the same column succeeds" {
skip_nbf_dolt_1
dolt sql <<SQL
CREATE TABLE test (
pk BIGINT NOT NULL COMMENT 'tag:0',
integration-tests/bats/default-values.bats
-1
View File
@@ -3,7 +3,6 @@ load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
setup_common
skip_nbf_dolt_1
dolt sql <<SQL
CREATE TABLE parent (
integration-tests/bats/helper/common.bash
+8
View File
@@ -24,6 +24,11 @@ set_dolt_user() {
dolt config --global --add user.email "$2" > /dev/null 2>&1
}
unset_dolt_user() {
dolt config --global --unset user.name
dolt config --global --unset user.email
}
current_dolt_user_name() {
dolt config --global --get user.name
}
@@ -52,6 +57,9 @@ assert_feature_version() {
# command, don't check its output in that case
if [ "$status" -eq 0 ]; then
[[ "$output" =~ "feature version: 3" ]] || exit 1
else
# Clear status to avoid BATS failing if this is the last run command
status=0
fi
}
integration-tests/bats/init.bats
+159
View File
@@ -0,0 +1,159 @@
#!/usr/bin/env bats
load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
setup_no_dolt_init
stash_current_dolt_user
}
teardown() {
restore_stashed_dolt_user
assert_feature_version
teardown_common
}
@test "init: implicit global configuration" {
set_dolt_user "baz", "baz@bash.com"
run dolt init
[ "$status" -eq 0 ]
run dolt config --local --get user.name
[ "$status" -eq 1 ]
run dolt config --local --get user.email
[ "$status" -eq 1 ]
assert_valid_repository
}
@test "init: explicit local configuration for name" {
set_dolt_user "baz", "baz@bash.com"
run dolt init --name foo
[ "$status" -eq 0 ]
run dolt config --local --get user.name
[ "$status" -eq 0 ]
[[ "$output" =~ "foo" ]] || false
run dolt config --local --get user.email
[ "$status" -eq 1 ]
assert_valid_repository
}
@test "init: explicit local configuration for email" {
set_dolt_user "baz", "baz@bash.com"
run dolt init --email foo@bar.com
[ "$status" -eq 0 ]
run dolt config --local --get user.name
[ "$status" -eq 1 ]
run dolt config --local --get user.email
[ "$status" -eq 0 ]
[[ "$output" =~ "foo@bar.com" ]] || false
assert_valid_repository
}
@test "init: explicit local configuration for name and email" {
set_dolt_user "baz", "baz@bash.com"
run dolt init --name foo --email foo@bar.com
[ "$status" -eq 0 ]
run dolt config --local --get user.name
[ "$status" -eq 0 ]
[[ "$output" =~ "foo" ]] || false
run dolt config --local --get user.email
[ "$status" -eq 0 ]
[[ "$output" =~ "foo@bar.com" ]] || false
assert_valid_repository
}
@test "init: explicit local configuration for name and email with no global config" {
unset_dolt_user
run dolt init --name foo --email foo@bar.com
[ "$status" -eq 0 ]
run dolt config --local --get user.name
[ "$status" -eq 0 ]
[[ "$output" =~ "foo" ]] || false
run dolt config --local --get user.email
[ "$status" -eq 0 ]
[[ "$output" =~ "foo@bar.com" ]] || false
assert_valid_repository
}
@test "init: no explicit or implicit configuration for name and email" {
unset_dolt_user
run dolt init
[ "$status" -eq 1 ]
[[ "$output" =~ "Author identity unknown" ]] || false
}
@test "init: implicit default initial branch" {
set_dolt_user "baz", "baz@bash.com"
run dolt init
[ "$status" -eq 0 ]
run dolt branch --show-current
[ "$status" -eq 0 ]
[[ "$output" =~ "main" ]] || false
assert_valid_repository
}
@test "init: implicit global initial branch" {
set_dolt_user "baz", "baz@bash.com"
run dolt config --global -add init.defaultbranch globalInitialBranch
run dolt init
[ "$status" -eq 0 ]
run dolt branch --show-current
[ "$status" -eq 0 ]
[[ "$output" =~ "globalInitialBranch" ]] || false
assert_valid_repository
}
@test "init: explicit initial branch" {
set_dolt_user "baz", "baz@bash.com"
run dolt init -b initialBranch
[ "$status" -eq 0 ]
run dolt branch --show-current
[ "$status" -eq 0 ]
[[ "$output" =~ "initialBranch" ]] || false
assert_valid_repository
}
@test "init: running init in existing Dolt directory fails" {
set_dolt_user "baz", "baz@bash.com"
run dolt init
[ "$status" -eq 0 ]
run dolt init
[ "$status" -eq 1 ]
}
assert_valid_repository () {
run dolt log
[ "$status" -eq 0 ]
[[ "$output" =~ "Initialize data repository" ]] || false
}
integration-tests/bats/sql-merge.bats
+3 -3
View File
@@ -444,7 +444,7 @@ SQL
[[ "$output" =~ "true" ]] || false
[[ "$output" =~ "true" ]] || false
[[ "${lines[1]}" =~ "DOLT_MERGE('feature-branch')" ]] || false # validate that merge returns 1 not "Updating..."
[[ "${lines[3]}" =~ "1" ]] || false
[[ "${lines[3]}" =~ "0" ]] || false
! [[ "$output" =~ "Updating" ]] || false
run dolt sql -q "SELECT * FROM test" -r csv
@@ -764,7 +764,7 @@ SQL
[[ "${lines[2]}" =~ "table,num_conflicts" ]] || false
[[ "${lines[3]}" =~ "one_pk,1" ]] || false
[[ "${lines[4]}" =~ "DOLT_MERGE('--abort')" ]] || false
[[ "${lines[5]}" =~ "1" ]] || false
[[ "${lines[5]}" =~ "0" ]] || false
# now resolve commits
run dolt sql << SQL
@@ -1308,7 +1308,7 @@ rollback;
SQL
[ $status -eq 0 ]
[[ "$output" =~ "| DOLT_MERGE('feature-branch') |" ]] || false
[[ "$output" =~ "| 0 |" ]] || false # conflict should return 0
[[ "$output" =~ "| 1 |" ]] || false # conflict should return 1
[[ "$output" =~ "| Warning | 1105 | merge has unresolved conflicts or constraint violations |" ]] || false
[[ "$output" =~ "| COUNT(*) |" ]] || false
[[ "$output" =~ "| 1 |" ]] || false
integration-tests/bats/status.bats
-1
View File
@@ -426,7 +426,6 @@ SQL
}
@test "status: roots runs even if status fails" {
skip_nbf_dolt_1
mv .dolt/repo_state.json .dolt/repo_state.backup
run dolt status
integration-tests/bats/tableplus.bats
+23
View File
@@ -221,3 +221,26 @@ SQL
run dolt sql -q "show full tables from test"
[ "$status" -eq 0 ]
}
@test "tableplus: information_schema.routines excludes procedure aliases, but works with CALL" {
run dolt sql -q "SELECT ROUTINE_SCHEMA as function_schema,ROUTINE_NAME as function_name,ROUTINE_DEFINITION as create_statement,ROUTINE_TYPE as function_type FROM information_schema.routines where ROUTINE_SCHEMA='test';"
[ "$status" -eq 0 ]
[[ ! "$output" =~ "dadd" ]] || false
[[ ! "$output" =~ "dadd" ]] || false
[[ ! "$output" =~ "dbranch" ]] || false
[[ ! "$output" =~ "dcheckout" ]] || false
[[ ! "$output" =~ "dclean" ]] || false
[[ ! "$output" =~ "dcommit" ]] || false
[[ ! "$output" =~ "dfetch" ]] || false
[[ ! "$output" =~ "dmerge" ]] || false
[[ ! "$output" =~ "dpull" ]] || false
[[ ! "$output" =~ "dpush" ]] || false
[[ ! "$output" =~ "dreset" ]] || false
[[ ! "$output" =~ "drevert" ]] || false
[[ ! "$output" =~ "dverify_constraints" ]] || false
[[ ! "$output" =~ "dverify_all_constraints" ]] || false
run dolt sql -q "CALL dbranch('branch1')"
[ "$status" -eq 0 ]
[[ "$output" =~ "status" ]] || false
}
integration-tests/mysql-client-tests/java/MySQLConnectorTest.java
+1 -1
View File
@@ -52,7 +52,7 @@ public class MySQLConnectorTest {
"1",
"1",
"0",
"1",
"0",
"3"
};
integration-tests/mysql-client-tests/node/index.js
+1 -1
View File
@@ -109,7 +109,7 @@ async function main() {
},
[],
[ { "dolt_checkout('main')": 0 } ],
[ { "dolt_merge('mybranch')": 1 } ],
[ { "dolt_merge('mybranch')": 0 } ],
[ { "COUNT(*)": 3 } ],
];
integration-tests/mysql-client-tests/python/mysql.connector-test.py
+1 -1
View File
@@ -23,7 +23,7 @@ QUERY_RESPONSE = [
{"insert into test (pk, `value`) values (1,1)": []},
{"select dolt_commit('-a', '-m', 'my commit2')": [('',)]},
{"select dolt_checkout('main')": [(0,)]},
{"select dolt_merge('mybranch')": [(1,)]},
{"select dolt_merge('mybranch')": [(0,)]},
{"select COUNT(*) FROM dolt_log": [(3,)]},
]
integration-tests/mysql-client-tests/python/pymysql-test.py
+1 -1
View File
@@ -16,7 +16,7 @@ QUERY_RESPONSE = [
{"insert into test (pk, `value`) values (1,1)": ()},
{"select dolt_commit('-a', '-m', 'my commit2')": (('',),)},
{"select dolt_checkout('main')": ((0,),)},
{"select dolt_merge('mybranch')": ((1,),)},
{"select dolt_merge('mybranch')": ((0,),)},
{"select COUNT(*) FROM dolt_log": ((3,),)},
]
integration-tests/mysql-client-tests/python/sqlalchemy-test.py
+1 -1
View File
@@ -20,7 +20,7 @@ QUERY_RESPONSE = [
{"insert into test (pk, `value`) values (1,1)": []},
{"select dolt_commit('-a', '-m', 'my commit2')": [('',)]},
{"select dolt_checkout('main')": [(0,)]},
{"select dolt_merge('mybranch')": [(1,)]},
{"select dolt_merge('mybranch')": [(0,)]},
{"select COUNT(*) FROM dolt_log": [(3,)]},
]