mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-04-23 21:59:01 -05:00
Code Issues Packages Projects Releases 157 Wiki Activity

doltdb: Remove some shims from VRW to NodeStore; carry NodeStores around.

Browse Source
This commit is contained in:
Aaron Son
2022-07-05 11:39:06 -07:00
parent 6229f6acb3
commit 829981ad64
53 changed files with 352 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/cmd/dolt/commands/cherry-pick.go
+1 -1
View File
@@ -254,7 +254,7 @@ func getParentAndCherryRoots(ctx context.Context, ddb *doltdb.DoltDB, cherryComm
return nil, nil, err
}
} else {
parentRoot, err = doltdb.EmptyRootValue(ctx, ddb.ValueReadWriter())
parentRoot, err = doltdb.EmptyRootValue(ctx, ddb.ValueReadWriter(), ddb.NodeStore())
if err != nil {
return nil, nil, err
}
go/cmd/dolt/commands/cvcmds/verify_constraints.go
+1 -1
View File
@@ -92,7 +92,7 @@ func (cmd VerifyConstraintsCmd) Exec(ctx context.Context, commandStr string, arg
return commands.HandleVErrAndExitCode(errhand.BuildDError("Unable to get head root.").AddCause(err).Build(), nil)
}
if verifyAllRows {
comparingRoot, err = doltdb.EmptyRootValue(ctx, comparingRoot.VRW())
comparingRoot, err = doltdb.EmptyRootValue(ctx, comparingRoot.VRW(), comparingRoot.NodeStore())
if err != nil {
return commands.HandleVErrAndExitCode(errhand.BuildDError("Unable to create an empty root.").AddCause(err).Build(), nil)
}
go/cmd/dolt/commands/schcmds/import.go
+3 -3
View File
@@ -296,7 +296,7 @@ func importSchema(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPars
return errhand.BuildDError("error: failed to get table.").AddCause(err).Build()
}
empty, err := durable.NewEmptyIndex(ctx, root.VRW(), sch)
empty, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), sch)
if err != nil {
return errhand.BuildDError("error: failed to get table.").AddCause(err).Build()
}
@@ -308,13 +308,13 @@ func importSchema(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPars
return errhand.BuildDError("error: failed to create table.").AddCause(err).Build()
}
} else {
indexSet, err = durable.NewIndexSetWithEmptyIndexes(ctx, root.VRW(), sch)
indexSet, err = durable.NewIndexSetWithEmptyIndexes(ctx, root.VRW(), root.NodeStore(), sch)
if err != nil {
return errhand.BuildDError("error: failed to get table.").AddCause(err).Build()
}
}
tbl, err = doltdb.NewTable(ctx, root.VRW(), sch, empty, indexSet, nil)
tbl, err = doltdb.NewTable(ctx, root.VRW(), root.NodeStore(), sch, empty, indexSet, nil)
if err != nil {
return errhand.BuildDError("error: failed to get table.").AddCause(err).Build()
}
go/libraries/doltcore/dbfactory/aws.go
+5 -3
View File
@@ -30,6 +30,7 @@ import (
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/nbs"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -107,18 +108,19 @@ type AWSFactory struct {
}
// CreateDB creates an AWS backed database
func (fact AWSFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, error) {
func (fact AWSFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
var db datas.Database
cs, err := fact.newChunkStore(ctx, nbf, urlObj, params)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
vrw := types.NewValueStore(cs)
db = datas.NewTypesDatabase(vrw)
ns := tree.NewNodeStore(cs)
return db, vrw, nil
return db, vrw, ns, nil
}
func (fact AWSFactory) newChunkStore(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (chunks.ChunkStore, error) {
go/libraries/doltcore/dbfactory/factory.go
+5 -4
View File
@@ -23,6 +23,7 @@ import (
"github.com/dolthub/dolt/go/libraries/utils/earl"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/prolly/tree"
)
const (
@@ -53,7 +54,7 @@ const (
// DBFactory is an interface for creating concrete datas.Database instances which may have different backing stores.
type DBFactory interface {
CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, error)
CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error)
}
// DBFactories is a map from url scheme name to DBFactory. Additional factories can be added to the DBFactories map
@@ -70,11 +71,11 @@ var DBFactories = map[string]DBFactory{
// CreateDB creates a database based on the supplied urlStr, and creation params. The DBFactory used for creation is
// determined by the scheme of the url. Naked urls will use https by default.
func CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlStr string, params map[string]interface{}) (datas.Database, types.ValueReadWriter, error) {
func CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlStr string, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
urlObj, err := earl.Parse(urlStr)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
scheme := urlObj.Scheme
@@ -86,5 +87,5 @@ func CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlStr string, para
return fact.CreateDB(ctx, nbf, urlObj, params)
}
return nil, nil, fmt.Errorf("unknown url scheme: '%s'", urlObj.Scheme)
return nil, nil, nil, fmt.Errorf("unknown url scheme: '%s'", urlObj.Scheme)
}
go/libraries/doltcore/dbfactory/factory_test.go
+2 -1
View File
@@ -54,9 +54,10 @@ func TestCreateFileDB(t *testing.T) {
func TestCreateMemDB(t *testing.T) {
ctx := context.Background()
db, vrw, err := CreateDB(ctx, types.Format_Default, "mem://", nil)
db, vrw, ns, err := CreateDB(ctx, types.Format_Default, "mem://", nil)
assert.NoError(t, err)
assert.NotNil(t, db)
assert.NotNil(t, vrw)
assert.NotNil(t, ns)
}
go/libraries/doltcore/dbfactory/file.go
+10 -8
View File
@@ -24,6 +24,7 @@ import (
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/nbs"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -43,11 +44,11 @@ type FileFactory struct {
}
// CreateDB creates an local filesys backed database
func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, error) {
func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
path, err := url.PathUnescape(urlObj.Path)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
path = filepath.FromSlash(path)
@@ -55,40 +56,41 @@ func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat,
err = validateDir(path)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
q := nbs.NewUnlimitedMemQuotaProvider()
newGenSt, err := nbs.NewLocalStore(ctx, nbf.VersionString(), path, defaultMemTableSize, q)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
oldgenPath := filepath.Join(path, "oldgen")
err = validateDir(oldgenPath)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return nil, nil, err
return nil, nil, nil, err
}
err = os.Mkdir(oldgenPath, os.ModePerm)
if err != nil && !errors.Is(err, os.ErrExist) {
return nil, nil, err
return nil, nil, nil, err
}
}
oldGenSt, err := nbs.NewLocalStore(ctx, newGenSt.Version(), oldgenPath, defaultMemTableSize, q)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
st := nbs.NewGenerationalCS(oldGenSt, newGenSt)
// metrics?
vrw := types.NewValueStore(st)
ns := tree.NewNodeStore(st)
return datas.NewTypesDatabase(vrw), vrw, nil
return datas.NewTypesDatabase(vrw), vrw, ns, nil
}
func validateDir(path string) error {
go/libraries/doltcore/dbfactory/grpc.go
+7 -5
View File
@@ -28,6 +28,7 @@ import (
"github.com/dolthub/dolt/go/libraries/events"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -51,28 +52,29 @@ func NewDoltRemoteFactory(insecure bool) DoltRemoteFactory {
// CreateDB creates a database backed by a remote server that implements the GRPC rpcs defined by
// remoteapis.ChunkStoreServiceClient
func (fact DoltRemoteFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, error) {
func (fact DoltRemoteFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
var db datas.Database
dpi, ok := params[GRPCDialProviderParam]
if dpi == nil || !ok {
return nil, nil, errors.New("DoltRemoteFactory.CreateDB must provide a GRPCDialProvider param through GRPCDialProviderParam")
return nil, nil, nil, errors.New("DoltRemoteFactory.CreateDB must provide a GRPCDialProvider param through GRPCDialProviderParam")
}
dp, ok := dpi.(GRPCDialProvider)
if !ok {
return nil, nil, errors.New("DoltRemoteFactory.CreateDB must provide a GRPCDialProvider param through GRPCDialProviderParam")
return nil, nil, nil, errors.New("DoltRemoteFactory.CreateDB must provide a GRPCDialProvider param through GRPCDialProviderParam")
}
cs, err := fact.newChunkStore(ctx, nbf, urlObj, params, dp)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
vrw := types.NewValueStore(cs)
db = datas.NewTypesDatabase(vrw)
ns := tree.NewNodeStore(cs)
return db, vrw, err
return db, vrw, ns, err
}
var NoCachingParameter = "__dolt__NO_CACHING"
go/libraries/doltcore/dbfactory/gs.go
+11 -8
View File
@@ -24,6 +24,7 @@ import (
"github.com/dolthub/dolt/go/store/blobstore"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/nbs"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -32,12 +33,12 @@ type GSFactory struct {
}
// CreateDB creates an GCS backed database
func (fact GSFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, error) {
func (fact GSFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
var db datas.Database
gcs, err := storage.NewClient(ctx)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
bs := blobstore.NewGCSBlobstore(gcs, urlObj.Host, urlObj.Path)
@@ -45,13 +46,14 @@ func (fact GSFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, ur
gcsStore, err := nbs.NewBSStore(ctx, nbf.VersionString(), bs, defaultMemTableSize, q)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
vrw := types.NewValueStore(gcsStore)
db = datas.NewTypesDatabase(vrw)
ns := tree.NewNodeStore(gcsStore)
return db, vrw, nil
return db, vrw, ns, nil
}
// LocalBSFactory is a DBFactory implementation for creating a local filesystem blobstore backed databases for testing
@@ -59,12 +61,12 @@ type LocalBSFactory struct {
}
// CreateDB creates a local filesystem blobstore backed database
func (fact LocalBSFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, error) {
func (fact LocalBSFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
var db datas.Database
absPath, err := filepath.Abs(filepath.Join(urlObj.Host, urlObj.Path))
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
bs := blobstore.NewLocalBlobstore(absPath)
@@ -72,11 +74,12 @@ func (fact LocalBSFactory) CreateDB(ctx context.Context, nbf *types.NomsBinForma
bsStore, err := nbs.NewBSStore(ctx, nbf.VersionString(), bs, defaultMemTableSize, q)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
vrw := types.NewValueStore(bsStore)
db = datas.NewTypesDatabase(vrw)
ns := tree.NewNodeStore(bsStore)
return db, vrw, err
return db, vrw, ns, err
}
go/libraries/doltcore/dbfactory/mem.go
+4 -2
View File
@@ -20,6 +20,7 @@ import (
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -28,12 +29,13 @@ type MemFactory struct {
}
// CreateDB creates an in memory backed database
func (fact MemFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, error) {
func (fact MemFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
var db datas.Database
storage := &chunks.MemoryStorage{}
cs := storage.NewViewWithDefaultFormat()
vrw := types.NewValueStore(cs)
db = datas.NewTypesDatabase(vrw)
ns := tree.NewNodeStore(cs)
return db, vrw, nil
return db, vrw, ns, nil
}
go/libraries/doltcore/diff/docs_diff_test.go
+6 -5
View File
@@ -26,6 +26,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
filesys2 "github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -57,7 +58,7 @@ func TestDocDiff(t *testing.T) {
sch := createTestDocsSchema()
licRow := makeDocRow(t, sch, doltdocs.LicenseDoc, types.String("license row"))
m, _ := createTestRows(t, ddb.ValueReadWriter(), sch, []row.Row{licRow})
tbl1, err := CreateTestTable(ddb.ValueReadWriter(), sch, m)
tbl1, err := CreateTestTable(ddb.ValueReadWriter(), ddb.NodeStore(), sch, m)
assert.NoError(t, err)
// Create root2 with tbl1 on it (one doc: license)
@@ -75,7 +76,7 @@ func TestDocDiff(t *testing.T) {
// Create tbl2 with one readme row
readmeRow := makeDocRow(t, sch, doltdocs.ReadmeDoc, types.String("readme row"))
m, _ = createTestRows(t, ddb.ValueReadWriter(), sch, []row.Row{readmeRow})
tbl2, err := CreateTestTable(ddb.ValueReadWriter(), sch, m)
tbl2, err := CreateTestTable(ddb.ValueReadWriter(), ddb.NodeStore(), sch, m)
assert.NoError(t, err)
// Create root3 with tbl2 on it (one doc: readme)
@@ -93,7 +94,7 @@ func TestDocDiff(t *testing.T) {
// Create tbl3 with 2 doc rows (readme, license)
readmeRowUpdated := makeDocRow(t, sch, doltdocs.ReadmeDoc, types.String("a different readme"))
m, _ = createTestRows(t, ddb.ValueReadWriter(), sch, []row.Row{readmeRowUpdated, licRow})
tbl3, err := CreateTestTable(ddb.ValueReadWriter(), sch, m)
tbl3, err := CreateTestTable(ddb.ValueReadWriter(), ddb.NodeStore(), sch, m)
assert.NoError(t, err)
// Create root4 with tbl3 on it (two docs: readme and license)
@@ -117,8 +118,8 @@ func TestDocDiff(t *testing.T) {
}
}
func CreateTestTable(vrw types.ValueReadWriter, tSchema schema.Schema, rowData types.Map) (*doltdb.Table, error) {
tbl, err := doltdb.NewNomsTable(context.Background(), vrw, tSchema, rowData, nil, nil)
func CreateTestTable(vrw types.ValueReadWriter, ns tree.NodeStore, tSchema schema.Schema, rowData types.Map) (*doltdb.Table, error) {
tbl, err := doltdb.NewNomsTable(context.Background(), vrw, ns, tSchema, rowData, nil, nil)
if err != nil {
return nil, err
go/libraries/doltcore/diff/table_deltas.go
+2 -2
View File
@@ -367,7 +367,7 @@ func (td TableDelta) GetRowData(ctx context.Context) (from, to durable.Index, er
return from, to, err
}
} else {
from, _ = durable.NewEmptyIndex(ctx, td.ToTable.ValueReadWriter(), td.ToSch)
from, _ = durable.NewEmptyIndex(ctx, td.ToTable.ValueReadWriter(), td.ToTable.NodeStore(), td.ToSch)
}
if td.ToTable != nil {
@@ -376,7 +376,7 @@ func (td TableDelta) GetRowData(ctx context.Context) (from, to durable.Index, er
return from, to, err
}
} else {
to, _ = durable.NewEmptyIndex(ctx, td.FromTable.ValueReadWriter(), td.FromSch)
to, _ = durable.NewEmptyIndex(ctx, td.FromTable.ValueReadWriter(), td.FromTable.NodeStore(), td.FromSch)
}
return from, to, nil
go/libraries/doltcore/doltdb/commit.go
+7 -5
View File
@@ -21,6 +21,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -30,16 +31,17 @@ var errHasNoRootValue = errors.New("no root value")
// Commit contains information on a commit that was written to noms
type Commit struct {
vrw types.ValueReadWriter
ns tree.NodeStore
parents []*datas.Commit
dCommit *datas.Commit
}
func NewCommit(ctx context.Context, vrw types.ValueReadWriter, commit *datas.Commit) (*Commit, error) {
func NewCommit(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, commit *datas.Commit) (*Commit, error) {
parents, err := datas.GetCommitParents(ctx, vrw, commit.NomsValue())
if err != nil {
return nil, err
}
return &Commit{vrw, parents, commit}, nil
return &Commit{vrw, ns, parents, commit}, nil
}
// HashOf returns the hash of the commit
@@ -84,11 +86,11 @@ func (c *Commit) GetRootValue(ctx context.Context) (*RootValue, error) {
if rootV == nil {
return nil, errHasNoRootValue
}
return newRootValue(c.vrw, rootV)
return newRootValue(c.vrw, c.ns, rootV)
}
func (c *Commit) GetParent(ctx context.Context, idx int) (*Commit, error) {
return NewCommit(ctx, c.vrw, c.parents[idx])
return NewCommit(ctx, c.vrw, c.ns, c.parents[idx])
}
var ErrNoCommonAncestor = errors.New("no common ancestor")
@@ -104,7 +106,7 @@ func GetCommitAncestor(ctx context.Context, cm1, cm2 *Commit) (*Commit, error) {
return nil, err
}
return NewCommit(ctx, cm1.vrw, targetCommit)
return NewCommit(ctx, cm1.vrw, cm1.ns, targetCommit)
}
func getCommitAncestorAddr(ctx context.Context, c1, c2 *datas.Commit, vrw1, vrw2 types.ValueReadWriter) (hash.Hash, error) {
go/libraries/doltcore/doltdb/commit_hooks_test.go
+2 -2
View File
@@ -106,7 +106,7 @@ func TestPushOnWriteHook(t *testing.T) {
tSchema := createTestSchema(t)
rowData, _ := createTestRowData(t, ddb.vrw, tSchema)
tbl, err := CreateTestTable(ddb.vrw, tSchema, rowData)
tbl, err := CreateTestTable(ddb.vrw, ddb.ns, tSchema, rowData)
if err != nil {
t.Fatal("Failed to create test table with data")
@@ -244,7 +244,7 @@ func TestAsyncPushOnWrite(t *testing.T) {
tSchema := createTestSchema(t)
rowData, _ := createTestRowData(t, ddb.vrw, tSchema)
tbl, err := CreateTestTable(ddb.vrw, tSchema, rowData)
tbl, err := CreateTestTable(ddb.vrw, ddb.ns, tSchema, rowData)
if err != nil {
t.Fatal("Failed to create test table with data")
go/libraries/doltcore/doltdb/commit_itr.go
+4 -3
View File
@@ -20,6 +20,7 @@ import (
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -132,7 +133,7 @@ func (cmItr *commitItr) Next(ctx context.Context) (hash.Hash, *Commit, error) {
next := cmItr.unprocessed[numUnprocessed-1]
cmItr.unprocessed = cmItr.unprocessed[:numUnprocessed-1]
cmItr.curr, err = hashToCommit(ctx, cmItr.ddb.ValueReadWriter(), next)
cmItr.curr, err = hashToCommit(ctx, cmItr.ddb.ValueReadWriter(), cmItr.ddb.ns, next)
if err != nil {
return hash.Hash{}, nil, err
@@ -141,12 +142,12 @@ func (cmItr *commitItr) Next(ctx context.Context) (hash.Hash, *Commit, error) {
return next, cmItr.curr, nil
}
func hashToCommit(ctx context.Context, vrw types.ValueReadWriter, h hash.Hash) (*Commit, error) {
func hashToCommit(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, h hash.Hash) (*Commit, error) {
dc, err := datas.LoadCommitAddr(ctx, vrw, h)
if err != nil {
return nil, err
}
return NewCommit(ctx, vrw, dc)
return NewCommit(ctx, vrw, ns, dc)
}
// CommitFilter is a function that returns true if a commit should be filtered out, and false if it should be kept
go/libraries/doltcore/doltdb/doltdb.go
+19 -12
View File
@@ -30,6 +30,7 @@ import (
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/datas/pull"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/types/edits"
)
@@ -66,14 +67,16 @@ var ErrCannotDeleteLastBranch = errors.New("cannot delete the last branch")
type DoltDB struct {
db hooksDatabase
vrw types.ValueReadWriter
ns tree.NodeStore
}
// DoltDBFromCS creates a DoltDB from a noms chunks.ChunkStore
func DoltDBFromCS(cs chunks.ChunkStore) *DoltDB {
vrw := types.NewValueStore(cs)
db := datas.NewTypesDatabase(vrw)
ns := tree.NewNodeStore(cs)
return &DoltDB{hooksDatabase{Database: db}, vrw}
return &DoltDB{hooksDatabase{Database: db}, vrw, ns}
}
// LoadDoltDB will acquire a reference to the underlying noms db. If the Location is InMemDoltDB then a reference
@@ -101,13 +104,13 @@ func LoadDoltDBWithParams(ctx context.Context, nbf *types.NomsBinFormat, urlStr
urlStr = fmt.Sprintf("file://%s", filepath.ToSlash(absPath))
}
db, vrw, err := dbfactory.CreateDB(ctx, nbf, urlStr, params)
db, vrw, ns, err := dbfactory.CreateDB(ctx, nbf, urlStr, params)
if err != nil {
return nil, err
}
return &DoltDB{hooksDatabase{Database: db}, vrw}, nil
return &DoltDB{hooksDatabase{Database: db}, vrw, ns}, nil
}
// NomsRoot returns the hash of the noms dataset map
@@ -162,7 +165,7 @@ func (ddb *DoltDB) WriteEmptyRepoWithCommitTimeAndDefaultBranch(
return errors.New("database already exists")
}
rv, err := EmptyRootValue(ctx, ddb.vrw)
rv, err := EmptyRootValue(ctx, ddb.vrw, ddb.ns)
if err != nil {
return err
@@ -318,7 +321,7 @@ func (ddb *DoltDB) Resolve(ctx context.Context, cs *CommitSpec, cwb ref.DoltRef)
return nil, err
}
commit, err := NewCommit(ctx, ddb.vrw, commitVal)
commit, err := NewCommit(ctx, ddb.vrw, ddb.ns, commitVal)
if err != nil {
return nil, err
}
@@ -332,7 +335,7 @@ func (ddb *DoltDB) ResolveCommitRef(ctx context.Context, ref ref.DoltRef) (*Comm
if err != nil {
return nil, err
}
return NewCommit(ctx, ddb.vrw, commitVal)
return NewCommit(ctx, ddb.vrw, ddb.ns, commitVal)
}
// ResolveTag takes a TagRef and returns the corresponding Tag object.
@@ -350,7 +353,7 @@ func (ddb *DoltDB) ResolveTag(ctx context.Context, tagRef ref.TagRef) (*Tag, err
return nil, fmt.Errorf("tagRef head is not a tag")
}
return NewTag(ctx, tagRef.GetPath(), ds, ddb.vrw)
return NewTag(ctx, tagRef.GetPath(), ds, ddb.vrw, ddb.ns)
}
// ResolveWorkingSet takes a WorkingSetRef and returns the corresponding WorkingSet object.
@@ -369,7 +372,7 @@ func (ddb *DoltDB) ResolveWorkingSet(ctx context.Context, workingSetRef ref.Work
return nil, fmt.Errorf("workingSetRef head is not a workingSetRef")
}
return NewWorkingSet(ctx, workingSetRef.GetPath(), ddb.vrw, ds)
return NewWorkingSet(ctx, workingSetRef.GetPath(), ddb.vrw, ddb.ns, ds)
}
// TODO: convenience method to resolve the head commit of a branch.
@@ -407,7 +410,7 @@ func (ddb *DoltDB) ReadRootValue(ctx context.Context, h hash.Hash) (*RootValue,
if err != nil {
return nil, err
}
return decodeRootNomsValue(ddb.vrw, val)
return decodeRootNomsValue(ddb.vrw, ddb.ns, val)
}
// Commit will update a branch's head value to be that of a previously committed root value hash
@@ -548,7 +551,7 @@ func (ddb *DoltDB) CommitWithParentCommits(ctx context.Context, valHash hash.Has
return nil, err
}
return NewCommit(ctx, ddb.vrw, dc)
return NewCommit(ctx, ddb.vrw, ddb.ns, dc)
}
// dangling commits are unreferenced by any branch or ref. They are created in the course of programmatic updates
@@ -582,7 +585,7 @@ func (ddb *DoltDB) CommitDanglingWithParentCommits(ctx context.Context, valHash
return nil, err
}
return NewCommit(ctx, ddb.vrw, dcommit)
return NewCommit(ctx, ddb.vrw, ddb.ns, dcommit)
}
// ValueReadWriter returns the underlying noms database as a types.ValueReadWriter.
@@ -590,6 +593,10 @@ func (ddb *DoltDB) ValueReadWriter() types.ValueReadWriter {
return ddb.vrw
}
func (ddb *DoltDB) NodeStore() tree.NodeStore {
return ddb.ns
}
func (ddb *DoltDB) Format() *types.NomsBinFormat {
return ddb.vrw.Format()
}
@@ -1011,7 +1018,7 @@ func (ddb *DoltDB) CommitWithWorkingSet(
return nil, err
}
return NewCommit(ctx, ddb.vrw, dc)
return NewCommit(ctx, ddb.vrw, ddb.ns, dc)
}
// DeleteWorkingSet deletes the working set given
go/libraries/doltcore/doltdb/doltdb_test.go
+4 -3
View File
@@ -32,6 +32,7 @@ import (
"github.com/dolthub/dolt/go/libraries/utils/test"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -69,8 +70,8 @@ func createTestSchema(t *testing.T) schema.Schema {
return sch
}
func CreateTestTable(vrw types.ValueReadWriter, tSchema schema.Schema, rowData types.Map) (*Table, error) {
tbl, err := NewNomsTable(context.Background(), vrw, tSchema, rowData, nil, nil)
func CreateTestTable(vrw types.ValueReadWriter, ns tree.NodeStore, tSchema schema.Schema, rowData types.Map) (*Table, error) {
tbl, err := NewNomsTable(context.Background(), vrw, ns, tSchema, rowData, nil, nil)
if err != nil {
return nil, err
@@ -283,7 +284,7 @@ func TestLDNoms(t *testing.T) {
tSchema := createTestSchema(t)
rowData, _ := createTestRowData(t, ddb.vrw, tSchema)
tbl, err = CreateTestTable(ddb.vrw, tSchema, rowData)
tbl, err = CreateTestTable(ddb.vrw, ddb.ns, tSchema, rowData)
if err != nil {
t.Fatal("Failed to create test table with data")
go/libraries/doltcore/doltdb/durable/artifact_index.go
+4 -6
View File
@@ -55,14 +55,13 @@ func RefFromArtifactIndex(ctx context.Context, vrw types.ValueReadWriter, idx Ar
}
// NewEmptyArtifactIndex returns an ArtifactIndex with no artifacts.
func NewEmptyArtifactIndex(ctx context.Context, vrw types.ValueReadWriter, tableSch schema.Schema) (ArtifactIndex, error) {
func NewEmptyArtifactIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, tableSch schema.Schema) (ArtifactIndex, error) {
switch vrw.Format() {
case types.Format_LD_1, types.Format_7_18, types.Format_DOLT_DEV:
panic("TODO")
case types.Format_DOLT_1:
kd := shim.KeyDescriptorFromSchema(tableSch)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
m, err := prolly.NewArtifactMapFromTuples(ctx, ns, kd)
if err != nil {
return nil, err
@@ -84,11 +83,11 @@ func ProllyMapFromArtifactIndex(i ArtifactIndex) prolly.ArtifactMap {
return i.(prollyArtifactIndex).index
}
func artifactIndexFromRef(ctx context.Context, vrw types.ValueReadWriter, tableSch schema.Schema, r types.Ref) (ArtifactIndex, error) {
return artifactIndexFromAddr(ctx, vrw, tableSch, r.TargetHash())
func artifactIndexFromRef(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, tableSch schema.Schema, r types.Ref) (ArtifactIndex, error) {
return artifactIndexFromAddr(ctx, vrw, ns, tableSch, r.TargetHash())
}
func artifactIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, tableSch schema.Schema, addr hash.Hash) (ArtifactIndex, error) {
func artifactIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, tableSch schema.Schema, addr hash.Hash) (ArtifactIndex, error) {
v, err := vrw.ReadValue(ctx, addr)
if err != nil {
return nil, err
@@ -101,7 +100,6 @@ func artifactIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, table
case types.Format_DOLT_1:
root := shim.NodeFromValue(v)
kd := shim.KeyDescriptorFromSchema(tableSch)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
m := prolly.NewArtifactMap(root, ns, kd)
return ArtifactIndexFromProllyMap(m), nil
go/libraries/doltcore/doltdb/durable/conflict_index.go
+1 -2
View File
@@ -47,7 +47,7 @@ func RefFromConflictIndex(ctx context.Context, vrw types.ValueReadWriter, idx Co
}
// NewEmptyConflictIndex returns an ConflictIndex with no rows.
func NewEmptyConflictIndex(ctx context.Context, vrw types.ValueReadWriter, oursSch, theirsSch, baseSch schema.Schema) (ConflictIndex, error) {
func NewEmptyConflictIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, oursSch, theirsSch, baseSch schema.Schema) (ConflictIndex, error) {
switch vrw.Format() {
case types.Format_LD_1, types.Format_7_18, types.Format_DOLT_DEV:
m, err := types.NewMap(ctx, vrw)
@@ -60,7 +60,6 @@ func NewEmptyConflictIndex(ctx context.Context, vrw types.ValueReadWriter, oursS
kd, oursVD := shim.MapDescriptorsFromSchema(oursSch)
theirsVD := shim.ValueDescriptorFromSchema(theirsSch)
baseVD := shim.ValueDescriptorFromSchema(baseSch)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
m := prolly.NewEmptyConflictMap(ns, kd, oursVD, theirsVD, baseVD)
go/libraries/doltcore/doltdb/durable/index.go
+5 -7
View File
@@ -114,7 +114,7 @@ func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, sch schema.Sc
}
// NewEmptyIndex returns an index with no rows.
func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema) (Index, error) {
func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) (Index, error) {
switch vrw.Format() {
case types.Format_LD_1, types.Format_7_18, types.Format_DOLT_DEV:
m, err := types.NewMap(ctx, vrw)
@@ -125,7 +125,6 @@ func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, sch schema.Sc
case types.Format_DOLT_1:
kd, vd := shim.MapDescriptorsFromSchema(sch)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
m, err := prolly.NewMapFromTuples(ctx, ns, kd, vd)
if err != nil {
return nil, err
@@ -318,9 +317,8 @@ func (i prollyIndex) AddColumnToRows(ctx context.Context, newCol string, newSche
}
// NewIndexSet returns an empty IndexSet.
func NewIndexSet(ctx context.Context, vrw types.ValueReadWriter) IndexSet {
func NewIndexSet(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore) IndexSet {
if vrw.Format().UsesFlatbuffers() {
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
emptyam := prolly.NewEmptyAddressMap(ns)
return doltDevIndexSet{vrw, emptyam}
}
@@ -332,10 +330,10 @@ func NewIndexSet(ctx context.Context, vrw types.ValueReadWriter) IndexSet {
}
}
func NewIndexSetWithEmptyIndexes(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema) (IndexSet, error) {
s := NewIndexSet(ctx, vrw)
func NewIndexSetWithEmptyIndexes(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) (IndexSet, error) {
s := NewIndexSet(ctx, vrw, ns)
for _, index := range sch.Indexes().AllIndexes() {
empty, err := NewEmptyIndex(ctx, vrw, index.Schema())
empty, err := NewEmptyIndex(ctx, vrw, ns, index.Schema())
if err != nil {
return nil, err
}
go/libraries/doltcore/doltdb/durable/table.go
+52 -41
View File
@@ -112,6 +112,7 @@ type Table interface {
type nomsTable struct {
vrw types.ValueReadWriter
ns tree.NodeStore
tableStruct types.Struct
}
@@ -120,14 +121,14 @@ var _ Table = nomsTable{}
var sharePool = pool.NewBuffPool()
// NewNomsTable makes a new Table.
func NewNomsTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema, rows types.Map, indexes IndexSet, autoIncVal types.Value) (Table, error) {
return NewTable(ctx, vrw, sch, nomsIndex{index: rows, vrw: vrw}, indexes, autoIncVal)
func NewNomsTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows types.Map, indexes IndexSet, autoIncVal types.Value) (Table, error) {
return NewTable(ctx, vrw, ns, sch, nomsIndex{index: rows, vrw: vrw}, indexes, autoIncVal)
}
// NewTable returns a new Table.
func NewTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema, rows Index, indexes IndexSet, autoIncVal types.Value) (Table, error) {
func NewTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows Index, indexes IndexSet, autoIncVal types.Value) (Table, error) {
if vrw.Format().UsesFlatbuffers() {
return newDoltDevTable(ctx, vrw, sch, rows, indexes, autoIncVal)
return newDoltDevTable(ctx, vrw, ns, sch, rows, indexes, autoIncVal)
}
schVal, err := encoding.MarshalSchemaAsNomsValue(ctx, vrw, sch)
@@ -146,7 +147,7 @@ func NewTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema,
}
if indexes == nil {
indexes = NewIndexSet(ctx, vrw)
indexes = NewIndexSet(ctx, vrw, ns)
}
indexesRef, err := refFromNomsValue(ctx, vrw, mapFromIndexSet(indexes))
@@ -169,11 +170,11 @@ func NewTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema,
return nil, err
}
return nomsTable{vrw, tableStruct}, nil
return nomsTable{vrw, ns, tableStruct}, nil
}
// TableFromAddr deserializes the table in the chunk at |addr|.
func TableFromAddr(ctx context.Context, vrw types.ValueReadWriter, addr hash.Hash) (Table, error) {
func TableFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, addr hash.Hash) (Table, error) {
val, err := vrw.ReadValue(ctx, addr)
if err != nil {
return nil, err
@@ -186,7 +187,7 @@ func TableFromAddr(ctx context.Context, vrw types.ValueReadWriter, addr hash.Has
return nil, err
}
return nomsTable{vrw: vrw, tableStruct: st}, nil
return nomsTable{vrw: vrw, tableStruct: st, ns: ns}, nil
} else {
sm, ok := val.(types.SerialMessage)
if !ok {
@@ -197,7 +198,7 @@ func TableFromAddr(ctx context.Context, vrw types.ValueReadWriter, addr hash.Has
err = errors.New("table ref is unexpected noms value; GetFileID == " + serial.GetFileID([]byte(sm)))
return nil, err
}
return doltDevTable{vrw, serial.GetRootAsTable([]byte(sm), 0)}, nil
return doltDevTable{vrw, ns, serial.GetRootAsTable([]byte(sm), 0)}, nil
}
}
@@ -224,6 +225,15 @@ func VrwFromTable(t Table) types.ValueReadWriter {
}
}
func NodeStoreFromTable(t Table) tree.NodeStore {
if nt, ok := t.(nomsTable); ok {
return nt.ns
} else {
ddt := t.(doltDevTable)
return ddt.ns
}
}
// valueReadWriter returns the valueReadWriter for this table.
func (t nomsTable) valueReadWriter() types.ValueReadWriter {
return t.vrw
@@ -277,7 +287,7 @@ func (t nomsTable) SetSchema(ctx context.Context, sch schema.Schema) (Table, err
return nil, err
}
return nomsTable{t.vrw, newTableStruct}, nil
return nomsTable{t.vrw, t.ns, newTableStruct}, nil
}
// SetTableRows implements Table.
@@ -292,7 +302,7 @@ func (t nomsTable) SetTableRows(ctx context.Context, updatedRows Index) (Table,
return nil, err
}
return nomsTable{t.vrw, updatedSt}, nil
return nomsTable{t.vrw, t.ns, updatedSt}, nil
}
// GetTableRows implements Table.
@@ -317,7 +327,7 @@ func (t nomsTable) GetIndexes(ctx context.Context) (IndexSet, error) {
return nil, err
}
if !ok {
return NewIndexSet(ctx, t.vrw), nil
return NewIndexSet(ctx, t.vrw, t.ns), nil
}
im, err := iv.(types.Ref).TargetValue(ctx, t.vrw)
@@ -334,7 +344,7 @@ func (t nomsTable) GetIndexes(ctx context.Context) (IndexSet, error) {
// SetIndexes implements Table.
func (t nomsTable) SetIndexes(ctx context.Context, indexes IndexSet) (Table, error) {
if indexes == nil {
indexes = NewIndexSet(ctx, t.vrw)
indexes = NewIndexSet(ctx, t.vrw, t.ns)
}
indexesRef, err := refFromNomsValue(ctx, t.vrw, mapFromIndexSet(indexes))
@@ -347,7 +357,7 @@ func (t nomsTable) SetIndexes(ctx context.Context, indexes IndexSet) (Table, err
return nil, err
}
return nomsTable{t.vrw, newTableStruct}, nil
return nomsTable{t.vrw, t.ns, newTableStruct}, nil
}
// GetArtifacts implements Table.
@@ -366,10 +376,10 @@ func (t nomsTable) GetArtifacts(ctx context.Context) (ArtifactIndex, error) {
return nil, err
}
if !ok {
return NewEmptyArtifactIndex(ctx, t.vrw, sch)
return NewEmptyArtifactIndex(ctx, t.vrw, t.ns, sch)
}
return artifactIndexFromRef(ctx, t.vrw, sch, val.(types.Ref))
return artifactIndexFromRef(ctx, t.vrw, t.ns, sch, val.(types.Ref))
}
// SetArtifacts implements Table.
@@ -388,7 +398,7 @@ func (t nomsTable) SetArtifacts(ctx context.Context, artifacts ArtifactIndex) (T
return nil, err
}
return nomsTable{t.vrw, updated}, nil
return nomsTable{t.vrw, t.ns, updated}, nil
}
// HasConflicts implements Table.
@@ -408,7 +418,7 @@ func (t nomsTable) GetConflicts(ctx context.Context) (conflict.ConflictSchema, C
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
empty, err := NewEmptyConflictIndex(ctx, t.vrw, sch, sch, sch)
empty, err := NewEmptyConflictIndex(ctx, t.vrw, t.ns, sch, sch, sch)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
@@ -426,7 +436,7 @@ func (t nomsTable) GetConflicts(ctx context.Context) (conflict.ConflictSchema, C
}
if conflictsVal == nil {
confIndex, err := NewEmptyConflictIndex(ctx, t.vrw, schemas.Schema, schemas.MergeSchema, schemas.Base)
confIndex, err := NewEmptyConflictIndex(ctx, t.vrw, t.ns, schemas.Schema, schemas.MergeSchema, schemas.Base)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
@@ -467,7 +477,7 @@ func (t nomsTable) SetConflicts(ctx context.Context, schemas conflict.ConflictSc
return nil, err
}
return nomsTable{t.vrw, updatedSt}, nil
return nomsTable{t.vrw, t.ns, updatedSt}, nil
}
// GetConflictSchemas implements Table.
@@ -519,7 +529,7 @@ func (t nomsTable) ClearConflicts(ctx context.Context) (Table, error) {
return nil, err
}
return nomsTable{t.vrw, tSt}, nil
return nomsTable{t.vrw, t.ns, tSt}, nil
}
// GetConstraintViolations implements Table.
@@ -547,7 +557,7 @@ func (t nomsTable) SetConstraintViolations(ctx context.Context, violationsMap ty
if err != nil {
return nil, err
}
return nomsTable{t.vrw, updatedStruct}, nil
return nomsTable{t.vrw, t.ns, updatedStruct}, nil
}
constraintViolationsRef, err := refFromNomsValue(ctx, t.vrw, violationsMap)
if err != nil {
@@ -557,7 +567,7 @@ func (t nomsTable) SetConstraintViolations(ctx context.Context, violationsMap ty
if err != nil {
return nil, err
}
return nomsTable{t.vrw, updatedStruct}, nil
return nomsTable{t.vrw, t.ns, updatedStruct}, nil
}
// GetAutoIncrement implements Table.
@@ -590,7 +600,7 @@ func (t nomsTable) SetAutoIncrement(ctx context.Context, val uint64) (Table, err
if err != nil {
return nil, err
}
return nomsTable{t.vrw, st}, nil
return nomsTable{t.vrw, t.ns, st}, nil
}
func (t nomsTable) DebugString(ctx context.Context) string {
@@ -689,6 +699,7 @@ func schemaFromAddr(ctx context.Context, vrw types.ValueReadWriter, addr hash.Ha
type doltDevTable struct {
vrw types.ValueReadWriter
ns tree.NodeStore
msg *serial.Table
}
@@ -764,7 +775,7 @@ func (fields serialTableFields) write() *serial.Table {
return serial.GetRootAsTable(builder.FinishedBytes(), 0)
}
func newDoltDevTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema, rows Index, indexes IndexSet, autoIncVal types.Value) (Table, error) {
func newDoltDevTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows Index, indexes IndexSet, autoIncVal types.Value) (Table, error) {
schVal, err := encoding.MarshalSchemaAsNomsValue(ctx, vrw, sch)
if err != nil {
return nil, err
@@ -782,7 +793,7 @@ func newDoltDevTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.
}
if indexes == nil {
indexes = NewIndexSet(ctx, vrw)
indexes = NewIndexSet(ctx, vrw, ns)
}
var autoInc uint64
@@ -804,7 +815,7 @@ func newDoltDevTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.
autoincval: autoInc,
}.write()
return doltDevTable{vrw, msg}, nil
return doltDevTable{vrw, ns, msg}, nil
}
func (t doltDevTable) nomsValue() types.Value {
@@ -842,7 +853,7 @@ func (t doltDevTable) SetSchema(ctx context.Context, sch schema.Schema) (Table,
addr := schRef.TargetHash()
msg := t.clone()
copy(msg.SchemaBytes(), addr[:])
return doltDevTable{t.vrw, msg}, nil
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) GetTableRows(ctx context.Context) (Index, error) {
@@ -874,13 +885,13 @@ func (t doltDevTable) SetTableRows(ctx context.Context, rows Index) (Table, erro
fields.rows = rowsbytes
msg := fields.write()
return doltDevTable{t.vrw, msg}, nil
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) GetIndexes(ctx context.Context) (IndexSet, error) {
ambytes := t.msg.SecondaryIndexesBytes()
node := tree.NodeFromBytes(ambytes)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(t.vrw))
ns := t.ns
return doltDevIndexSet{t.vrw, prolly.NewAddressMap(node, ns)}, nil
}
@@ -888,7 +899,7 @@ func (t doltDevTable) SetIndexes(ctx context.Context, indexes IndexSet) (Table,
fields := t.fields()
fields.indexes = indexes.(doltDevIndexSet).am
msg := fields.write()
return doltDevTable{t.vrw, msg}, nil
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) GetConflicts(ctx context.Context) (conflict.ConflictSchema, ConflictIndex, error) {
@@ -903,7 +914,7 @@ func (t doltDevTable) GetConflicts(ctx context.Context) (conflict.ConflictSchema
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
empty, err := NewEmptyConflictIndex(ctx, t.vrw, sch, sch, sch)
empty, err := NewEmptyConflictIndex(ctx, t.vrw, t.ns, sch, sch, sch)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
@@ -932,7 +943,7 @@ func (t doltDevTable) GetConflicts(ctx context.Context) (conflict.ConflictSchema
mapaddr := hash.New(conflicts.DataBytes())
var conflictIdx ConflictIndex
if mapaddr.IsEmpty() {
conflictIdx, err = NewEmptyConflictIndex(ctx, t.vrw, ourschema, theirschema, baseschema)
conflictIdx, err = NewEmptyConflictIndex(ctx, t.vrw, t.ns, ourschema, theirschema, baseschema)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
@@ -959,10 +970,10 @@ func (t doltDevTable) GetArtifacts(ctx context.Context) (ArtifactIndex, error) {
addr := hash.New(t.msg.ArtifactsBytes())
if addr.IsEmpty() {
return NewEmptyArtifactIndex(ctx, t.vrw, sch)
return NewEmptyArtifactIndex(ctx, t.vrw, t.ns, sch)
}
return artifactIndexFromAddr(ctx, t.vrw, sch, addr)
return artifactIndexFromAddr(ctx, t.vrw, t.ns, sch, addr)
}
// SetArtifacts implements Table.
@@ -981,7 +992,7 @@ func (t doltDevTable) SetArtifacts(ctx context.Context, artifacts ArtifactIndex)
}
msg := t.clone()
copy(msg.ArtifactsBytes(), addr[:])
return doltDevTable{t.vrw, msg}, nil
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) HasConflicts(ctx context.Context) (bool, error) {
@@ -1018,7 +1029,7 @@ func (t doltDevTable) SetConflicts(ctx context.Context, sch conflict.ConflictSch
copy(cmsg.TheirSchemaBytes(), theiraddr[:])
copy(cmsg.AncestorSchemaBytes(), baseaddr[:])
return doltDevTable{t.vrw, msg}, nil
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) ClearConflicts(ctx context.Context) (Table, error) {
@@ -1029,7 +1040,7 @@ func (t doltDevTable) ClearConflicts(ctx context.Context) (Table, error) {
copy(conflicts.OurSchemaBytes(), emptyhash[:])
copy(conflicts.TheirSchemaBytes(), emptyhash[:])
copy(conflicts.AncestorSchemaBytes(), emptyhash[:])
return doltDevTable{t.vrw, msg}, nil
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) GetConstraintViolations(ctx context.Context) (types.Map, error) {
@@ -1055,7 +1066,7 @@ func (t doltDevTable) SetConstraintViolations(ctx context.Context, violations ty
}
msg := t.clone()
copy(msg.ViolationsBytes(), addr[:])
return doltDevTable{t.vrw, msg}, nil
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) GetAutoIncrement(ctx context.Context) (uint64, error) {
@@ -1074,7 +1085,7 @@ func (t doltDevTable) SetAutoIncrement(ctx context.Context, val uint64) (Table,
fields.autoincval = val
msg = fields.write()
}
return doltDevTable{t.vrw, msg}, nil
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) clone() *serial.Table {
@@ -1088,7 +1099,7 @@ func (t doltDevTable) clone() *serial.Table {
func (t doltDevTable) fields() serialTableFields {
ambytes := t.msg.SecondaryIndexesBytes()
node := tree.NodeFromBytes(ambytes)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(t.vrw))
ns := t.ns
conflicts := t.msg.Conflicts(nil)
return serialTableFields{
go/libraries/doltcore/doltdb/root_val.go
+35 -32
View File
@@ -54,6 +54,7 @@ var DoltFeatureVersion FeatureVersion = 3 // last bumped when storing creation t
// RootValue is the value of the Database and is the committed value in every Dolt commit.
type RootValue struct {
vrw types.ValueReadWriter
ns tree.NodeStore
st rvStorage
fkc *ForeignKeyCollection // cache the first load
}
@@ -81,7 +82,7 @@ func tmIterAll(ctx context.Context, tm tableMap, cb func(name string, addr hash.
type rvStorage interface {
GetFeatureVersion() (FeatureVersion, bool, error)
GetTablesMap(ctx context.Context, vr types.ValueReadWriter) (tableMap, error)
GetTablesMap(ctx context.Context, vr types.ValueReadWriter, ns tree.NodeStore) (tableMap, error)
GetSuperSchemaMap(ctx context.Context, vr types.ValueReader) (types.Map, bool, error)
GetForeignKeys(ctx context.Context, vr types.ValueReader) (types.Value, bool, error)
@@ -89,7 +90,7 @@ type rvStorage interface {
SetForeignKeyMap(ctx context.Context, vrw types.ValueReadWriter, m types.Value) (rvStorage, error)
SetFeatureVersion(v FeatureVersion) (rvStorage, error)
EditTablesMap(ctx context.Context, vrw types.ValueReadWriter, edits []tableEdit) (rvStorage, error)
EditTablesMap(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, edits []tableEdit) (rvStorage, error)
DebugString(ctx context.Context) string
nomsValue() types.Value
@@ -111,7 +112,7 @@ func (r nomsRvStorage) GetFeatureVersion() (FeatureVersion, bool, error) {
}
}
func (r nomsRvStorage) GetTablesMap(context.Context, types.ValueReadWriter) (tableMap, error) {
func (r nomsRvStorage) GetTablesMap(context.Context, types.ValueReadWriter, tree.NodeStore) (tableMap, error) {
v, found, err := r.valueSt.MaybeGet(tablesKey)
if err != nil {
return nil, err
@@ -175,8 +176,8 @@ func (r nomsRvStorage) SetSuperSchemaMap(ctx context.Context, vrw types.ValueRea
return nomsRvStorage{st}, nil
}
func (r nomsRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWriter, edits []tableEdit) (rvStorage, error) {
m, err := r.GetTablesMap(ctx, vrw)
func (r nomsRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, edits []tableEdit) (rvStorage, error) {
m, err := r.GetTablesMap(ctx, vrw, ns)
if err != nil {
return nil, err
}
@@ -250,7 +251,7 @@ func (r nomsRvStorage) nomsValue() types.Value {
return r.valueSt
}
func newRootValue(vrw types.ValueReadWriter, v types.Value) (*RootValue, error) {
func newRootValue(vrw types.ValueReadWriter, ns tree.NodeStore, v types.Value) (*RootValue, error) {
var storage rvStorage
if vrw.Format().UsesFlatbuffers() {
@@ -277,20 +278,20 @@ func newRootValue(vrw types.ValueReadWriter, v types.Value) (*RootValue, error)
}
}
return &RootValue{vrw, storage, nil}, nil
return &RootValue{vrw, ns, storage, nil}, nil
}
// LoadRootValueFromRootIshAddr takes the hash of the commit or the hash of a
// working set and returns the corresponding RootValue.
func LoadRootValueFromRootIshAddr(ctx context.Context, vrw types.ValueReadWriter, h hash.Hash) (*RootValue, error) {
func LoadRootValueFromRootIshAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, h hash.Hash) (*RootValue, error) {
val, err := datas.LoadRootNomsValueFromRootIshAddr(ctx, vrw, h)
if err != nil {
return nil, err
}
return decodeRootNomsValue(vrw, val)
return decodeRootNomsValue(vrw, ns, val)
}
func decodeRootNomsValue(vrw types.ValueReadWriter, val types.Value) (*RootValue, error) {
func decodeRootNomsValue(vrw types.ValueReadWriter, ns tree.NodeStore, val types.Value) (*RootValue, error) {
if val == nil {
return nil, ErrNoRootValAtHash
}
@@ -299,7 +300,7 @@ func decodeRootNomsValue(vrw types.ValueReadWriter, val types.Value) (*RootValue
return nil, ErrNoRootValAtHash
}
return newRootValue(vrw, val)
return newRootValue(vrw, ns, val)
}
func isRootValue(nbf *types.NomsBinFormat, val types.Value) bool {
@@ -315,11 +316,10 @@ func isRootValue(nbf *types.NomsBinFormat, val types.Value) bool {
return false
}
func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter) (*RootValue, error) {
func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore) (*RootValue, error) {
if vrw.Format().UsesFlatbuffers() {
builder := flatbuffers.NewBuilder(80)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
emptyam := prolly.NewEmptyAddressMap(ns)
ambytes := []byte(tree.ValueFromNode(emptyam.Node()).(types.TupleRowStorage))
tablesoff := builder.CreateByteVector(ambytes)
@@ -334,7 +334,7 @@ func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter) (*RootValue,
serial.RootValueAddSuperSchemasAddr(builder, ssoff)
builder.FinishWithFileIdentifier(serial.RootValueEnd(builder), []byte(serial.RootValueFileID))
bs := builder.FinishedBytes()
return newRootValue(vrw, types.SerialMessage(bs))
return newRootValue(vrw, ns, types.SerialMessage(bs))
}
empty, err := types.NewMap(ctx, vrw)
@@ -354,13 +354,17 @@ func EmptyRootValue(ctx context.Context, vrw types.ValueReadWriter) (*RootValue,
return nil, err
}
return newRootValue(vrw, st)
return newRootValue(vrw, ns, st)
}
func (root *RootValue) VRW() types.ValueReadWriter {
return root.vrw
}
func (root *RootValue) NodeStore() tree.NodeStore {
return root.ns
}
// GetFeatureVersion returns the feature version of this root, if one is written
func (root *RootValue) GetFeatureVersion(ctx context.Context) (ver FeatureVersion, ok bool, err error) {
return root.st.GetFeatureVersion()
@@ -375,7 +379,7 @@ func (root *RootValue) setFeatureVersion(v FeatureVersion) (*RootValue, error) {
}
func (root *RootValue) HasTable(ctx context.Context, tName string) (bool, error) {
tableMap, err := root.st.GetTablesMap(ctx, root.vrw)
tableMap, err := root.st.GetTablesMap(ctx, root.vrw, root.ns)
if err != nil {
return false, err
}
@@ -671,7 +675,7 @@ func (root *RootValue) GetTable(ctx context.Context, tName string) (*Table, bool
return nil, false, nil
}
table, err := durable.TableFromAddr(ctx, root.VRW(), addr)
table, err := durable.TableFromAddr(ctx, root.VRW(), root.ns, addr)
if err != nil {
return nil, false, err
}
@@ -745,7 +749,7 @@ func (root *RootValue) GetTableNames(ctx context.Context) ([]string, error) {
}
func (root *RootValue) getTableMap(ctx context.Context) (tableMap, error) {
return root.st.GetTablesMap(ctx, root.vrw)
return root.st.GetTablesMap(ctx, root.vrw, root.ns)
}
func (root *RootValue) TablesInConflict(ctx context.Context) ([]string, error) {
@@ -827,7 +831,7 @@ func (root *RootValue) IterTables(ctx context.Context, cb func(name string, tabl
}
return tm.Iter(ctx, func(name string, addr hash.Hash) (bool, error) {
nt, err := durable.TableFromAddr(ctx, root.VRW(), addr)
nt, err := durable.TableFromAddr(ctx, root.VRW(), root.ns, addr)
if err != nil {
return true, err
}
@@ -896,7 +900,7 @@ func (root *RootValue) PutSuperSchema(ctx context.Context, tName string, ss *sch
}
func (root *RootValue) withStorage(st rvStorage) *RootValue {
return &RootValue{root.vrw, st, nil}
return &RootValue{root.vrw, root.ns, st, nil}
}
func (root *RootValue) nomsValue() types.Value {
@@ -923,7 +927,7 @@ func putTable(ctx context.Context, root *RootValue, tName string, ref types.Ref)
panic("Don't attempt to put a table with a name that fails the IsValidTableName check")
}
newStorage, err := root.st.EditTablesMap(ctx, root.vrw, []tableEdit{{name: tName, ref: &ref}})
newStorage, err := root.st.EditTablesMap(ctx, root.vrw, root.ns, []tableEdit{{name: tName, ref: &ref}})
if err != nil {
return nil, err
}
@@ -933,12 +937,12 @@ func putTable(ctx context.Context, root *RootValue, tName string, ref types.Ref)
// CreateEmptyTable creates an empty table in this root with the name and schema given, returning the new root value.
func (root *RootValue) CreateEmptyTable(ctx context.Context, tName string, sch schema.Schema) (*RootValue, error) {
empty, err := durable.NewEmptyIndex(ctx, root.vrw, sch)
empty, err := durable.NewEmptyIndex(ctx, root.vrw, root.ns, sch)
if err != nil {
return nil, err
}
indexes := durable.NewIndexSet(ctx, root.VRW())
indexes := durable.NewIndexSet(ctx, root.VRW(), root.ns)
err = sch.Indexes().Iter(func(index schema.Index) (stop bool, err error) {
// create an empty map for every index
indexes, err = indexes.PutIndex(ctx, index.Name(), empty)
@@ -948,7 +952,7 @@ func (root *RootValue) CreateEmptyTable(ctx context.Context, tName string, sch s
return nil, err
}
tbl, err := NewTable(ctx, root.VRW(), sch, empty, indexes, nil)
tbl, err := NewTable(ctx, root.VRW(), root.ns, sch, empty, indexes, nil)
if err != nil {
return nil, err
}
@@ -1040,7 +1044,7 @@ func (root *RootValue) UpdateSuperSchemasFromOther(ctx context.Context, tblNames
// column tag information, use this method instead of a table drop + add.
func (root *RootValue) RenameTable(ctx context.Context, oldName, newName string) (*RootValue, error) {
newStorage, err := root.st.EditTablesMap(ctx, root.vrw, []tableEdit{{old_name: oldName, name: newName}})
newStorage, err := root.st.EditTablesMap(ctx, root.vrw, root.ns, []tableEdit{{old_name: oldName, name: newName}})
if err != nil {
return nil, err
}
@@ -1089,7 +1093,7 @@ func (root *RootValue) RemoveTables(ctx context.Context, skipFKHandling bool, al
edits[i].name = name
}
newStorage, err := root.st.EditTablesMap(ctx, root.vrw, edits)
newStorage, err := root.st.EditTablesMap(ctx, root.vrw, root.ns, edits)
if err != nil {
return nil, err
}
@@ -1387,15 +1391,14 @@ func (r fbRvStorage) GetFeatureVersion() (FeatureVersion, bool, error) {
return FeatureVersion(r.srv.FeatureVersion()), true, nil
}
func (r fbRvStorage) getAddressMap(vrw types.ValueReadWriter) prolly.AddressMap {
func (r fbRvStorage) getAddressMap(vrw types.ValueReadWriter, ns tree.NodeStore) prolly.AddressMap {
tbytes := r.srv.TablesBytes()
node := shim.NodeFromValue(types.TupleRowStorage(tbytes))
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
return prolly.NewAddressMap(node, ns)
}
func (r fbRvStorage) GetTablesMap(ctx context.Context, vrw types.ValueReadWriter) (tableMap, error) {
am := r.getAddressMap(vrw)
func (r fbRvStorage) GetTablesMap(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore) (tableMap, error) {
am := r.getAddressMap(vrw, ns)
return fbTableMap{am}, nil
}
@@ -1457,10 +1460,10 @@ func (r fbRvStorage) SetSuperSchemaMap(ctx context.Context, vrw types.ValueReadW
return ret, nil
}
func (r fbRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWriter, edits []tableEdit) (rvStorage, error) {
func (r fbRvStorage) EditTablesMap(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, edits []tableEdit) (rvStorage, error) {
builder := flatbuffers.NewBuilder(80)
am := r.getAddressMap(vrw)
am := r.getAddressMap(vrw, ns)
ae := am.Editor()
for _, e := range edits {
if e.old_name != "" {
go/libraries/doltcore/doltdb/table.go
+9 -14
View File
@@ -29,7 +29,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -79,8 +78,8 @@ type Table struct {
}
// NewNomsTable creates a noms Struct which stores row data, index data, and schema.
func NewNomsTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema, rows types.Map, indexes durable.IndexSet, autoIncVal types.Value) (*Table, error) {
dt, err := durable.NewNomsTable(ctx, vrw, sch, rows, indexes, autoIncVal)
func NewNomsTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows types.Map, indexes durable.IndexSet, autoIncVal types.Value) (*Table, error) {
dt, err := durable.NewNomsTable(ctx, vrw, ns, sch, rows, indexes, autoIncVal)
if err != nil {
return nil, err
}
@@ -89,8 +88,8 @@ func NewNomsTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.Sch
}
// NewTable creates a durable object which stores row data, index data, and schema.
func NewTable(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema, rows durable.Index, indexes durable.IndexSet, autoIncVal types.Value) (*Table, error) {
dt, err := durable.NewTable(ctx, vrw, sch, rows, indexes, autoIncVal)
func NewTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows durable.Index, indexes durable.IndexSet, autoIncVal types.Value) (*Table, error) {
dt, err := durable.NewTable(ctx, vrw, ns, sch, rows, indexes, autoIncVal)
if err != nil {
return nil, err
}
@@ -108,12 +107,8 @@ func (t *Table) ValueReadWriter() types.ValueReadWriter {
return durable.VrwFromTable(t.table)
}
// NodeStore returns the NodeStore for this table.
func (t *Table) NodeStore() tree.NodeStore {
if t == nil {
return nil
}
return tree.NewNodeStore(shim.ChunkStoreFromVRW(t.ValueReadWriter()))
return durable.NodeStoreFromTable(t.table)
}
// SetConflicts sets the merge conflicts for this table.
@@ -279,11 +274,11 @@ func (t *Table) getProllyConflictSchemas(ctx context.Context, tblName string) (b
return nil, nil, nil, err
}
baseTbl, err := tableFromRootIsh(ctx, t.ValueReadWriter(), art.Metadata.BaseRootIsh, tblName)
baseTbl, err := tableFromRootIsh(ctx, t.ValueReadWriter(), t.NodeStore(), art.Metadata.BaseRootIsh, tblName)
if err != nil {
return nil, nil, nil, err
}
theirTbl, err := tableFromRootIsh(ctx, t.ValueReadWriter(), art.TheirRootIsh, tblName)
theirTbl, err := tableFromRootIsh(ctx, t.ValueReadWriter(), t.NodeStore(), art.TheirRootIsh, tblName)
if err != nil {
return nil, nil, nil, err
}
@@ -300,8 +295,8 @@ func (t *Table) getProllyConflictSchemas(ctx context.Context, tblName string) (b
return baseSch, ourSch, theirSch, nil
}
func tableFromRootIsh(ctx context.Context, vrw types.ValueReadWriter, h hash.Hash, tblName string) (*Table, error) {
rv, err := LoadRootValueFromRootIshAddr(ctx, vrw, h)
func tableFromRootIsh(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, h hash.Hash, tblName string) (*Table, error) {
rv, err := LoadRootValueFromRootIshAddr(ctx, vrw, ns, h)
if err != nil {
return nil, err
}
go/libraries/doltcore/doltdb/tag.go
+3 -2
View File
@@ -20,6 +20,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -32,7 +33,7 @@ type Tag struct {
}
// NewTag creates a new Tag object.
func NewTag(ctx context.Context, name string, ds datas.Dataset, vrw types.ValueReadWriter) (*Tag, error) {
func NewTag(ctx context.Context, name string, ds datas.Dataset, vrw types.ValueReadWriter, ns tree.NodeStore) (*Tag, error) {
meta, commitAddr, err := ds.HeadTag()
if err != nil {
return nil, err
@@ -41,7 +42,7 @@ func NewTag(ctx context.Context, name string, ds datas.Dataset, vrw types.ValueR
if err != nil {
return nil, err
}
commit, err := NewCommit(ctx, vrw, dc)
commit, err := NewCommit(ctx, vrw, ns, dc)
if err != nil {
return nil, err
}
go/libraries/doltcore/doltdb/workingset.go
+6 -5
View File
@@ -22,6 +22,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -128,7 +129,7 @@ func (ws WorkingSet) Meta() *datas.WorkingSetMeta {
}
// NewWorkingSet creates a new WorkingSet object.
func NewWorkingSet(ctx context.Context, name string, vrw types.ValueReadWriter, ds datas.Dataset) (*WorkingSet, error) {
func NewWorkingSet(ctx context.Context, name string, vrw types.ValueReadWriter, ns tree.NodeStore, ds datas.Dataset) (*WorkingSet, error) {
dsws, err := ds.HeadWorkingSet()
if err != nil {
return nil, err
@@ -148,7 +149,7 @@ func NewWorkingSet(ctx context.Context, name string, vrw types.ValueReadWriter,
if err != nil {
return nil, err
}
workingRoot, err := newRootValue(vrw, workingRootVal)
workingRoot, err := newRootValue(vrw, ns, workingRootVal)
if err != nil {
return nil, err
}
@@ -160,7 +161,7 @@ func NewWorkingSet(ctx context.Context, name string, vrw types.ValueReadWriter,
return nil, err
}
stagedRoot, err = newRootValue(vrw, stagedRootVal)
stagedRoot, err = newRootValue(vrw, ns, stagedRootVal)
if err != nil {
return nil, err
}
@@ -177,7 +178,7 @@ func NewWorkingSet(ctx context.Context, name string, vrw types.ValueReadWriter,
return nil, err
}
commit, err := NewCommit(ctx, vrw, fromDCommit)
commit, err := NewCommit(ctx, vrw, ns, fromDCommit)
if err != nil {
return nil, err
}
@@ -187,7 +188,7 @@ func NewWorkingSet(ctx context.Context, name string, vrw types.ValueReadWriter,
return nil, err
}
preMergeWorkingRoot, err := newRootValue(vrw, preMergeWorkingV)
preMergeWorkingRoot, err := newRootValue(vrw, ns, preMergeWorkingV)
if err != nil {
return nil, err
}
go/libraries/doltcore/doltdocs/docs_table.go
+4 -3
View File
@@ -23,6 +23,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/table"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -80,7 +81,7 @@ func updateDocsTable(ctx context.Context, docTbl *doltdb.Table, docs Docs) (*dol
}
// createDocsTable creates a new in memory table that stores the given doc details.
func createDocsTable(ctx context.Context, vrw types.ValueReadWriter, docs Docs) (*doltdb.Table, error) {
func createDocsTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, docs Docs) (*doltdb.Table, error) {
rows := make([]row.Row, 0, len(docs))
@@ -121,7 +122,7 @@ func createDocsTable(ctx context.Context, vrw types.ValueReadWriter, docs Docs)
return nil, err
}
newDocsTbl, err := doltdb.NewNomsTable(ctx, vrw, DocsSchema, rowMap, nil, nil)
newDocsTbl, err := doltdb.NewNomsTable(ctx, vrw, ns, DocsSchema, rowMap, nil, nil)
if err != nil {
return nil, err
}
@@ -139,7 +140,7 @@ func CreateOrUpdateDocsTable(ctx context.Context, root *doltdb.RootValue, docs D
if found {
return updateDocsTable(ctx, docsTbl, docs)
} else {
return createDocsTable(ctx, root.VRW(), docs)
return createDocsTable(ctx, root.VRW(), root.NodeStore(), docs)
}
}
go/libraries/doltcore/doltdocs/docs_test.go
+5 -4
View File
@@ -25,6 +25,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
filesys2 "github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -46,7 +47,7 @@ func TestAddNewerTextAndValueFromTable(t *testing.T) {
sch := createTestDocsSchema()
rows := []row.Row{}
m, _ := createTestRows(t, ddb.ValueReadWriter(), sch, rows)
tbl, err := CreateTestTable(ddb.ValueReadWriter(), sch, m)
tbl, err := CreateTestTable(ddb.ValueReadWriter(), ddb.NodeStore(), sch, m)
require.NoError(t, err)
// If a table doesn't have doc row, doc Text and Value should remain nil
@@ -66,7 +67,7 @@ func TestAddNewerTextAndValueFromTable(t *testing.T) {
// Update tbl to have 2 doc rows, readme and license
rows = getDocRows(t, sch, types.String("text in doc_text"))
m, _ = createTestRows(t, ddb.ValueReadWriter(), sch, rows)
tbl, err = CreateTestTable(ddb.ValueReadWriter(), sch, m)
tbl, err = CreateTestTable(ddb.ValueReadWriter(), ddb.NodeStore(), sch, m)
require.NoError(t, err)
// If a table has a doc row, Text and Value and should be updated to the `doc_text` value in that row.
@@ -133,8 +134,8 @@ func TestAddNewerTextAndDocPkFromRow(t *testing.T) {
assert.Equal(t, LicenseDoc, doc3.DocPk)
}
func CreateTestTable(vrw types.ValueReadWriter, tSchema schema.Schema, rowData types.Map) (*doltdb.Table, error) {
tbl, err := doltdb.NewNomsTable(context.Background(), vrw, tSchema, rowData, nil, nil)
func CreateTestTable(vrw types.ValueReadWriter, ns tree.NodeStore, tSchema schema.Schema, rowData types.Map) (*doltdb.Table, error) {
tbl, err := doltdb.NewNomsTable(context.Background(), vrw, ns, tSchema, rowData, nil, nil)
if err != nil {
return nil, err
go/libraries/doltcore/dtestutils/environment.go
+2 -1
View File
@@ -70,6 +70,7 @@ func CreateEnvWithSeedData(t *testing.T) *env.DoltEnv {
ctx := context.Background()
vrw := dEnv.DoltDB.ValueReadWriter()
ns := dEnv.DoltDB.NodeStore()
rowMap, err := types.NewMap(ctx, vrw)
require.NoError(t, err)
@@ -86,7 +87,7 @@ func CreateEnvWithSeedData(t *testing.T) *env.DoltEnv {
ai := sch.Indexes().AllIndexes()
sch.Indexes().Merge(ai...)
tbl, err := doltdb.NewNomsTable(ctx, vrw, sch, rowMap, nil, nil)
tbl, err := doltdb.NewNomsTable(ctx, vrw, ns, sch, rowMap, nil, nil)
require.NoError(t, err)
tbl, err = editor.RebuildAllIndexes(ctx, tbl, editor.TestEditorOptions(vrw))
require.NoError(t, err)
go/libraries/doltcore/dtestutils/schema.go
+4 -2
View File
@@ -102,6 +102,7 @@ func CreateTestTable(t *testing.T, dEnv *env.DoltEnv, tableName string, sch sche
ctx := context.Background()
vrw := dEnv.DoltDB.ValueReadWriter()
ns := dEnv.DoltDB.NodeStore()
rowMap, err := types.NewMap(ctx, vrw)
require.NoError(t, err)
@@ -115,7 +116,7 @@ func CreateTestTable(t *testing.T, dEnv *env.DoltEnv, tableName string, sch sche
rowMap, err = me.Map(ctx)
require.NoError(t, err)
tbl, err := doltdb.NewNomsTable(ctx, vrw, sch, rowMap, nil, nil)
tbl, err := doltdb.NewNomsTable(ctx, vrw, ns, sch, rowMap, nil, nil)
require.NoError(t, err)
tbl, err = editor.RebuildAllIndexes(ctx, tbl, editor.TestEditorOptions(vrw))
require.NoError(t, err)
@@ -137,7 +138,8 @@ func putTableToWorking(ctx context.Context, dEnv *env.DoltEnv, sch schema.Schema
}
vrw := dEnv.DoltDB.ValueReadWriter()
tbl, err := doltdb.NewNomsTable(ctx, vrw, sch, rows, indexData, autoVal)
ns := dEnv.DoltDB.NodeStore()
tbl, err := doltdb.NewNomsTable(ctx, vrw, ns, sch, rows, indexData, autoVal)
if err != nil {
return err
}
go/libraries/doltcore/dtestutils/testcommands/multienv.go
+4 -2
View File
@@ -330,6 +330,7 @@ func (mr *MultiRepoTestSetup) PushToRemote(dbName, remoteName, branchName string
func createTestTable(dEnv *env.DoltEnv, tableName string, sch schema.Schema, errhand func(args ...interface{}), rs ...row.Row) {
ctx := context.Background()
vrw := dEnv.DoltDB.ValueReadWriter()
ns := dEnv.DoltDB.NodeStore()
rowMap, err := types.NewMap(ctx, vrw)
if err != nil {
@@ -346,7 +347,7 @@ func createTestTable(dEnv *env.DoltEnv, tableName string, sch schema.Schema, err
errhand(err)
}
tbl, err := doltdb.NewNomsTable(ctx, vrw, sch, rowMap, nil, nil)
tbl, err := doltdb.NewNomsTable(ctx, vrw, ns, sch, rowMap, nil, nil)
if err != nil {
errhand(err)
}
@@ -380,7 +381,8 @@ func putTableToWorking(ctx context.Context, dEnv *env.DoltEnv, sch schema.Schema
}
vrw := dEnv.DoltDB.ValueReadWriter()
tbl, err := doltdb.NewNomsTable(ctx, vrw, sch, rows, indexData, autoVal)
ns := dEnv.DoltDB.NodeStore()
tbl, err := doltdb.NewNomsTable(ctx, vrw, ns, sch, rows, indexData, autoVal)
if err != nil {
return err
}
go/libraries/doltcore/merge/keyless_integration_test.go
+5 -4
View File
@@ -31,6 +31,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/pool"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
@@ -335,8 +336,8 @@ func assertProllyConflicts(t *testing.T, ctx context.Context, tbl *doltdb.Table,
c++
ours := mustGetRowValueFromTable(t, ctx, tbl, conf.Key)
theirs := mustGetRowValueFromRootIsh(t, ctx, tbl.ValueReadWriter(), conf.TheirRootIsh, tblName, conf.Key)
base := mustGetRowValueFromRootIsh(t, ctx, tbl.ValueReadWriter(), conf.Metadata.BaseRootIsh, tblName, conf.Key)
theirs := mustGetRowValueFromRootIsh(t, ctx, tbl.ValueReadWriter(), tbl.NodeStore(), conf.TheirRootIsh, tblName, conf.Key)
base := mustGetRowValueFromRootIsh(t, ctx, tbl.ValueReadWriter(), tbl.NodeStore(), conf.Metadata.BaseRootIsh, tblName, conf.Key)
copy(h[:], conf.Key.GetField(0))
expectedConf, ok := expectedSet[h]
@@ -401,8 +402,8 @@ func mustGetRowValueFromTable(t *testing.T, ctx context.Context, tbl *doltdb.Tab
return value
}
func mustGetRowValueFromRootIsh(t *testing.T, ctx context.Context, vrw types.ValueReadWriter, rootIsh hash.Hash, tblName string, key val.Tuple) val.Tuple {
rv, err := doltdb.LoadRootValueFromRootIshAddr(ctx, vrw, rootIsh)
func mustGetRowValueFromRootIsh(t *testing.T, ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, rootIsh hash.Hash, tblName string, key val.Tuple) val.Tuple {
rv, err := doltdb.LoadRootValueFromRootIshAddr(ctx, vrw, ns, rootIsh)
require.NoError(t, err)
tbl, ok, err := rv.GetTable(ctx, tblName)
require.NoError(t, err)
go/libraries/doltcore/merge/merge.go
+8 -5
View File
@@ -34,6 +34,7 @@ import (
"github.com/dolthub/dolt/go/libraries/utils/valutil"
"github.com/dolthub/dolt/go/store/atomicerr"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -60,11 +61,12 @@ type Merger struct {
mergeRoot *doltdb.RootValue
ancRoot *doltdb.RootValue
vrw types.ValueReadWriter
ns tree.NodeStore
}
// NewMerger creates a new merger utility object.
func NewMerger(ctx context.Context, theirRootIsh, ancRootIsh hash.Hash, root, mergeRoot, ancRoot *doltdb.RootValue, vrw types.ValueReadWriter) *Merger {
return &Merger{theirRootIsh, ancRootIsh, root, mergeRoot, ancRoot, vrw}
func NewMerger(ctx context.Context, theirRootIsh, ancRootIsh hash.Hash, root, mergeRoot, ancRoot *doltdb.RootValue, vrw types.ValueReadWriter, ns tree.NodeStore) *Merger {
return &Merger{theirRootIsh, ancRootIsh, root, mergeRoot, ancRoot, vrw, ns}
}
// MergeTable merges schema and table data for the table tblName.
@@ -118,11 +120,11 @@ func (merger *Merger) MergeTable(ctx context.Context, tblName string, opts edito
// If both added the same table, pretend it was in the ancestor all along with no data
// Don't touch ancHash to avoid triggering other short-circuit logic below
ancHasTable, ancSchema, ancTbl = true, rootSchema, tbl
ancRows, err = durable.NewEmptyIndex(ctx, merger.vrw, ancSchema)
ancRows, err = durable.NewEmptyIndex(ctx, merger.vrw, merger.ns, ancSchema)
if err != nil {
return nil, nil, err
}
ancIndexSet, err = durable.NewIndexSetWithEmptyIndexes(ctx, merger.vrw, ancSchema)
ancIndexSet, err = durable.NewIndexSetWithEmptyIndexes(ctx, merger.vrw, merger.ns, ancSchema)
if err != nil {
return nil, nil, err
}
@@ -208,6 +210,7 @@ func (merger *Merger) MergeTable(ctx context.Context, tblName string, opts edito
updatedTbl, stats, err = mergeTableData(
ctx,
merger.vrw,
merger.ns,
tblName,
postMergeSchema, rootSchema, mergeSchema, ancSchema,
tbl, mergeTbl, updatedTbl,
@@ -996,7 +999,7 @@ func MergeRoots(ctx context.Context, theirRootIsh, ancRootIsh hash.Hash, ourRoot
// Merge tables one at a time. This is done based on name. With table names from ourRoot being merged first,
// renaming a table will return delete/modify conflict error consistently.
// TODO: merge based on a more durable table identity that persists across renames
merger := NewMerger(ctx, theirRootIsh, ancRootIsh, ourRoot, theirRoot, ancRoot, ourRoot.VRW())
merger := NewMerger(ctx, theirRootIsh, ancRootIsh, ourRoot, theirRoot, ancRoot, ourRoot.VRW(), ourRoot.NodeStore())
for _, tblName := range tblNames {
mergedTable, stats, err := merger.MergeTable(ctx, tblName, opts, isCherryPick)
if err != nil {
go/libraries/doltcore/merge/merge_prolly.go
+2
View File
@@ -49,6 +49,7 @@ import (
func mergeTableData(
ctx context.Context,
vrw types.ValueReadWriter,
ns tree.NodeStore,
tblName string,
postMergeSchema, rootSchema, mergeSchema, ancSchema schema.Schema,
tbl, mergeTbl, updatedTbl *doltdb.Table,
@@ -130,6 +131,7 @@ func mergeTableData(
updatedTbl, err = mergeProllySecondaryIndexes(
ctx,
vrw,
ns,
postMergeSchema, rootSchema, mergeSchema, ancSchema,
mergedData,
tbl, mergeTbl, updatedTbl,
go/libraries/doltcore/merge/merge_prolly_indexes.go
+8 -4
View File
@@ -106,6 +106,7 @@ type confVals struct {
func mergeProllySecondaryIndexes(
ctx context.Context,
vrw types.ValueReadWriter,
ns tree.NodeStore,
postMergeSchema, rootSch, mergeSch, ancSch schema.Schema,
mergedData durable.Index,
tbl, mergeTbl, tableToUpdate *doltdb.Table,
@@ -125,6 +126,7 @@ func mergeProllySecondaryIndexes(
mergedSet, err := mergeProllyIndexSets(
ctx,
vrw,
ns,
postMergeSchema, rootSch, mergeSch, ancSch,
mergedData,
rootSet, mergeSet, ancSet,
@@ -146,13 +148,14 @@ func mergeProllySecondaryIndexes(
func mergeProllyIndexSets(
ctx context.Context,
vrw types.ValueReadWriter,
ns tree.NodeStore,
postMergeSchema, rootSch, mergeSch, ancSch schema.Schema,
mergedData durable.Index,
root, merge, anc durable.IndexSet,
artEditor prolly.ArtifactsEditor,
theirRootIsh hash.Hash,
tblName string) (durable.IndexSet, error) {
mergedIndexSet := durable.NewIndexSet(ctx, vrw)
mergedIndexSet := durable.NewIndexSet(ctx, vrw, ns)
mergedM := durable.ProllyMapFromIndex(mergedData)
@@ -191,7 +194,7 @@ func mergeProllyIndexSets(
mergedIndex, err := func() (durable.Index, error) {
if !rootOK || !mergeOK || !ancOK {
return buildIndex(ctx, vrw, postMergeSchema, index, mergedM, artEditor, theirRootIsh, tblName)
return buildIndex(ctx, vrw, ns, postMergeSchema, index, mergedM, artEditor, theirRootIsh, tblName)
}
if index.IsUnique() {
@@ -232,7 +235,7 @@ func mergeProllyIndexSets(
return mergedIndexSet, nil
}
func buildIndex(ctx context.Context, vrw types.ValueReadWriter, postMergeSchema schema.Schema, index schema.Index, m prolly.Map, artEditor prolly.ArtifactsEditor, theirRootIsh hash.Hash, tblName string) (durable.Index, error) {
func buildIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, postMergeSchema schema.Schema, index schema.Index, m prolly.Map, artEditor prolly.ArtifactsEditor, theirRootIsh hash.Hash, tblName string) (durable.Index, error) {
if index.IsUnique() {
meta, err := makeUniqViolMeta(postMergeSchema, index)
if err != nil {
@@ -249,6 +252,7 @@ func buildIndex(ctx context.Context, vrw types.ValueReadWriter, postMergeSchema
mergedMap, err := creation.BuildUniqueProllyIndex(
ctx,
vrw,
ns,
postMergeSchema,
index,
m,
@@ -271,7 +275,7 @@ func buildIndex(ctx context.Context, vrw types.ValueReadWriter, postMergeSchema
return mergedMap, nil
}
mergedIndex, err := creation.BuildSecondaryProllyIndex(ctx, vrw, postMergeSchema, index, m)
mergedIndex, err := creation.BuildSecondaryProllyIndex(ctx, vrw, ns, postMergeSchema, index, m)
if err != nil {
return nil, err
}
go/libraries/doltcore/merge/merge_test.go
+20 -20
View File
@@ -307,8 +307,8 @@ func TestMergeCommits(t *testing.T) {
t.Skip()
}
vrw, rightCommitHash, ancCommitHash, root, mergeRoot, ancRoot, expectedRows, expectedArtifacts := setupMergeTest(t)
merger := NewMerger(context.Background(), rightCommitHash, ancCommitHash, root, mergeRoot, ancRoot, vrw)
vrw, ns, rightCommitHash, ancCommitHash, root, mergeRoot, ancRoot, expectedRows, expectedArtifacts := setupMergeTest(t)
merger := NewMerger(context.Background(), rightCommitHash, ancCommitHash, root, mergeRoot, ancRoot, vrw, ns)
opts := editor.TestEditorOptions(vrw)
// TODO: stats
merged, _, err := merger.MergeTable(context.Background(), tableName, opts, false)
@@ -320,7 +320,7 @@ func TestMergeCommits(t *testing.T) {
assert.NoError(t, err)
sch, err := tbl.GetSchema(context.Background())
assert.NoError(t, err)
expected, err := doltdb.NewTable(context.Background(), vrw, sch, expectedRows, nil, nil)
expected, err := doltdb.NewTable(context.Background(), vrw, ns, sch, expectedRows, nil, nil)
assert.NoError(t, err)
expected, err = rebuildAllProllyIndexes(context.Background(), expected)
assert.NoError(t, err)
@@ -357,9 +357,9 @@ func TestNomsMergeCommits(t *testing.T) {
t.Skip()
}
vrw, rightCmHash, ancCmHash, root, mergeRoot, ancRoot, expectedRows, expectedConflicts, expectedStats := setupNomsMergeTest(t)
vrw, ns, rightCmHash, ancCmHash, root, mergeRoot, ancRoot, expectedRows, expectedConflicts, expectedStats := setupNomsMergeTest(t)
merger := NewMerger(context.Background(), rightCmHash, ancCmHash, root, mergeRoot, ancRoot, vrw)
merger := NewMerger(context.Background(), rightCmHash, ancCmHash, root, mergeRoot, ancRoot, vrw, ns)
opts := editor.TestEditorOptions(vrw)
merged, stats, err := merger.MergeTable(context.Background(), tableName, opts, false)
if err != nil {
@@ -371,7 +371,7 @@ func TestNomsMergeCommits(t *testing.T) {
assert.NoError(t, err)
sch, err := tbl.GetSchema(context.Background())
assert.NoError(t, err)
expected, err := doltdb.NewNomsTable(context.Background(), vrw, sch, expectedRows, nil, nil)
expected, err := doltdb.NewNomsTable(context.Background(), vrw, ns, sch, expectedRows, nil, nil)
assert.NoError(t, err)
expected, err = editor.RebuildAllIndexes(context.Background(), expected, editor.TestEditorOptions(vrw))
assert.NoError(t, err)
@@ -418,13 +418,12 @@ func sortTests(t []testRow) {
})
}
func setupMergeTest(t *testing.T) (types.ValueReadWriter, hash.Hash, hash.Hash, *doltdb.RootValue, *doltdb.RootValue, *doltdb.RootValue, durable.Index, prolly.ArtifactMap) {
func setupMergeTest(t *testing.T) (types.ValueReadWriter, tree.NodeStore, hash.Hash, hash.Hash, *doltdb.RootValue, *doltdb.RootValue, *doltdb.RootValue, durable.Index, prolly.ArtifactMap) {
ddb := mustMakeEmptyRepo(t)
vrw := ddb.ValueReadWriter()
ns := ddb.NodeStore()
sortTests(testRows)
ns := tree.NewNodeStore(shim.ChunkStoreFromVRW(vrw))
var initialKVs []val.Tuple
var expectedKVs []val.Tuple
@@ -474,17 +473,17 @@ func setupMergeTest(t *testing.T) (types.ValueReadWriter, hash.Hash, hash.Hash,
mergeRows, err := rightMut.Map(context.Background())
require.NoError(t, err)
rootTbl, err := doltdb.NewTable(context.Background(), vrw, sch, durable.IndexFromProllyMap(updatedRows), nil, nil)
rootTbl, err := doltdb.NewTable(context.Background(), vrw, ns, sch, durable.IndexFromProllyMap(updatedRows), nil, nil)
require.NoError(t, err)
rootTbl, err = rebuildAllProllyIndexes(context.Background(), rootTbl)
require.NoError(t, err)
mergeTbl, err := doltdb.NewTable(context.Background(), vrw, sch, durable.IndexFromProllyMap(mergeRows), nil, nil)
mergeTbl, err := doltdb.NewTable(context.Background(), vrw, ns, sch, durable.IndexFromProllyMap(mergeRows), nil, nil)
require.NoError(t, err)
mergeTbl, err = rebuildAllProllyIndexes(context.Background(), mergeTbl)
require.NoError(t, err)
ancTbl, err := doltdb.NewTable(context.Background(), vrw, sch, durable.IndexFromProllyMap(initialRows), nil, nil)
ancTbl, err := doltdb.NewTable(context.Background(), vrw, ns, sch, durable.IndexFromProllyMap(initialRows), nil, nil)
require.NoError(t, err)
ancTbl, err = rebuildAllProllyIndexes(context.Background(), ancTbl)
require.NoError(t, err)
@@ -511,12 +510,13 @@ func setupMergeTest(t *testing.T) (types.ValueReadWriter, hash.Hash, hash.Hash,
expectedArtifacts, err := artEditor.Flush(context.Background())
require.NoError(t, err)
return vrw, rightCmHash, baseCmHash, root, mergeRoot, ancRoot, durable.IndexFromProllyMap(expectedRows), expectedArtifacts
return vrw, ns, rightCmHash, baseCmHash, root, mergeRoot, ancRoot, durable.IndexFromProllyMap(expectedRows), expectedArtifacts
}
func setupNomsMergeTest(t *testing.T) (types.ValueReadWriter, hash.Hash, hash.Hash, *doltdb.RootValue, *doltdb.RootValue, *doltdb.RootValue, types.Map, types.Map, *MergeStats) {
func setupNomsMergeTest(t *testing.T) (types.ValueReadWriter, tree.NodeStore, hash.Hash, hash.Hash, *doltdb.RootValue, *doltdb.RootValue, *doltdb.RootValue, types.Map, types.Map, *MergeStats) {
ddb := mustMakeEmptyRepo(t)
vrw := ddb.ValueReadWriter()
ns := ddb.NodeStore()
sortTests(testRows)
var initalKVs []types.Value
@@ -575,29 +575,29 @@ func setupNomsMergeTest(t *testing.T) (types.ValueReadWriter, hash.Hash, hash.Ha
mergeRows, err := rightE.Map(context.Background())
require.NoError(t, err)
tbl, err := doltdb.NewNomsTable(context.Background(), vrw, sch, initialRows, nil, nil)
tbl, err := doltdb.NewNomsTable(context.Background(), vrw, ns, sch, initialRows, nil, nil)
require.NoError(t, err)
tbl, err = editor.RebuildAllIndexes(context.Background(), tbl, editor.TestEditorOptions(vrw))
require.NoError(t, err)
updatedTbl, err := doltdb.NewNomsTable(context.Background(), vrw, sch, updatedRows, nil, nil)
updatedTbl, err := doltdb.NewNomsTable(context.Background(), vrw, ns, sch, updatedRows, nil, nil)
require.NoError(t, err)
updatedTbl, err = editor.RebuildAllIndexes(context.Background(), updatedTbl, editor.TestEditorOptions(vrw))
require.NoError(t, err)
mergeTbl, err := doltdb.NewNomsTable(context.Background(), vrw, sch, mergeRows, nil, nil)
mergeTbl, err := doltdb.NewNomsTable(context.Background(), vrw, ns, sch, mergeRows, nil, nil)
require.NoError(t, err)
mergeTbl, err = editor.RebuildAllIndexes(context.Background(), mergeTbl, editor.TestEditorOptions(vrw))
require.NoError(t, err)
ancTable, err := doltdb.NewNomsTable(context.Background(), vrw, sch, initialRows, nil, nil)
ancTable, err := doltdb.NewNomsTable(context.Background(), vrw, ns, sch, initialRows, nil, nil)
require.NoError(t, err)
ancTable, err = editor.RebuildAllIndexes(context.Background(), ancTable, editor.TestEditorOptions(vrw))
require.NoError(t, err)
rightCmHash, ancCommitHash, root, mergeRoot, ancRoot := buildLeftRightAncCommitsAndBranches(t, ddb, updatedTbl, mergeTbl, ancTable)
return vrw, rightCmHash, ancCommitHash, root, mergeRoot, ancRoot, expectedRows, expectedConflicts, calcExpectedStats(t)
return vrw, ns, rightCmHash, ancCommitHash, root, mergeRoot, ancRoot, expectedRows, expectedConflicts, calcExpectedStats(t)
}
// rebuildAllProllyIndexes builds the data for the secondary indexes in |tbl|'s
@@ -624,7 +624,7 @@ func rebuildAllProllyIndexes(ctx context.Context, tbl *doltdb.Table) (*doltdb.Ta
primary := durable.ProllyMapFromIndex(tableRowData)
for _, index := range sch.Indexes().AllIndexes() {
rebuiltIndexRowData, err := creation.BuildSecondaryProllyIndex(ctx, tbl.ValueReadWriter(), sch, index, primary)
rebuiltIndexRowData, err := creation.BuildSecondaryProllyIndex(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), sch, index, primary)
if err != nil {
return nil, err
}
go/libraries/doltcore/merge/resolve.go
+4 -3
View File
@@ -28,6 +28,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
"github.com/dolthub/dolt/go/libraries/utils/set"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -41,7 +42,7 @@ func Theirs(key types.Value, cnf conflict.Conflict) (types.Value, error) {
return cnf.MergeValue, nil
}
func ResolveTable(ctx context.Context, vrw types.ValueReadWriter, tblName string, root *doltdb.RootValue, autoResFunc AutoResolver, opts editor.Options) (*doltdb.RootValue, error) {
func ResolveTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, tblName string, root *doltdb.RootValue, autoResFunc AutoResolver, opts editor.Options) (*doltdb.RootValue, error) {
tbl, ok, err := root.GetTable(ctx, tblName)
if err != nil {
return nil, err
@@ -75,7 +76,7 @@ func ResolveTable(ctx context.Context, vrw types.ValueReadWriter, tblName string
return nil, err
}
confIdx, err := durable.NewEmptyConflictIndex(ctx, vrw, schemas.Schema, schemas.MergeSchema, schemas.Base)
confIdx, err := durable.NewEmptyConflictIndex(ctx, vrw, ns, schemas.Schema, schemas.MergeSchema, schemas.Base)
if err != nil {
return nil, err
}
@@ -291,7 +292,7 @@ func autoResolve(ctx context.Context, dEnv *env.DoltEnv, root *doltdb.RootValue,
var err error
opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: dEnv.TempTableFilesDir()}
for _, tblName := range tbls {
root, err = ResolveTable(ctx, root.VRW(), tblName, root, autoResolver, opts)
root, err = ResolveTable(ctx, root.VRW(), root.NodeStore(), tblName, root, autoResolver, opts)
if err != nil {
return err
}
go/libraries/doltcore/merge/revert.go
+1 -1
View File
@@ -72,7 +72,7 @@ func Revert(ctx context.Context, ddb *doltdb.DoltDB, root *doltdb.RootValue, hea
return nil, "", err
}
} else {
theirRoot, err = doltdb.EmptyRootValue(ctx, ddb.ValueReadWriter())
theirRoot, err = doltdb.EmptyRootValue(ctx, ddb.ValueReadWriter(), ddb.NodeStore())
if err != nil {
return nil, "", err
}
go/libraries/doltcore/merge/violations_fk.go
+2 -2
View File
@@ -99,7 +99,7 @@ func AddForeignKeyViolations(ctx context.Context, newRoot, baseRoot *doltdb.Root
return nil, nil, err
}
// Parent does not exist in the ancestor so we use an empty map
emptyIdx, err := durable.NewEmptyIndex(ctx, postParent.Table.ValueReadWriter(), postParent.Schema)
emptyIdx, err := durable.NewEmptyIndex(ctx, postParent.Table.ValueReadWriter(), postParent.Table.NodeStore(), postParent.Schema)
if err != nil {
return nil, nil, err
}
@@ -122,7 +122,7 @@ func AddForeignKeyViolations(ctx context.Context, newRoot, baseRoot *doltdb.Root
}
innerFoundViolations := false
// Child does not exist in the ancestor so we use an empty map
emptyIdx, err := durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postChild.Schema)
emptyIdx, err := durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postChild.Table.NodeStore(), postChild.Schema)
if err != nil {
return nil, nil, err
}
go/libraries/doltcore/schema/encoding/schema_marshaling_test.go
+2 -2
View File
@@ -52,7 +52,7 @@ func createTestSchema() schema.Schema {
func TestNomsMarshalling(t *testing.T) {
tSchema := createTestSchema()
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
_, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
if err != nil {
t.Fatal("Could not create in mem noms db.")
@@ -158,7 +158,7 @@ func TestTypeInfoMarshalling(t *testing.T) {
nbf, err := types.GetFormatForVersionString(constants.FormatDefaultString)
require.NoError(t, err)
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), nbf, nil, nil)
_, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), nbf, nil, nil)
require.NoError(t, err)
val, err := MarshalSchemaAsNomsValue(context.Background(), vrw, originalSch)
require.NoError(t, err)
go/libraries/doltcore/sqle/alterschema.go
+1 -1
View File
@@ -427,7 +427,7 @@ func insertKeyedData(ctx context.Context, nbf *types.NomsBinFormat, oldTable *do
}
// Create the new Table and rebuild all the indexes
newTable, err := doltdb.NewNomsTable(ctx, oldTable.ValueReadWriter(), newSchema, empty, nil, nil)
newTable, err := doltdb.NewNomsTable(ctx, oldTable.ValueReadWriter(), oldTable.NodeStore(), newSchema, empty, nil, nil)
if err != nil {
return nil, err
}
go/libraries/doltcore/sqle/dfunctions/verify_constraints.go
+1 -1
View File
@@ -81,7 +81,7 @@ func DoDoltConstraintsVerify(ctx *sql.Context, args []string) (int, error) {
var comparingRoot *doltdb.RootValue
if verifyAll {
comparingRoot, err = doltdb.EmptyRootValue(ctx, workingRoot.VRW())
comparingRoot, err = doltdb.EmptyRootValue(ctx, workingRoot.VRW(), workingRoot.NodeStore())
if err != nil {
return 1, err
}
go/libraries/doltcore/sqle/dsess/transactions.go
+1 -1
View File
@@ -437,7 +437,7 @@ func (tx *DoltTransaction) stompConflicts(ctx *sql.Context, mergedRoot *doltdb.R
var err error
root := mergedRoot
for _, tblName := range tablesWithConflicts {
root, err = merge.ResolveTable(ctx, mergedRoot.VRW(), tblName, root, merge.Theirs, tx.mergeEditOpts)
root, err = merge.ResolveTable(ctx, mergedRoot.VRW(), mergedRoot.NodeStore(), tblName, root, merge.Theirs, tx.mergeEditOpts)
if err != nil {
return nil, err
}
go/libraries/doltcore/sqle/dtables/conflicts_tables_prolly.go
+5 -2
View File
@@ -28,6 +28,7 @@ import (
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/pool"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
@@ -106,6 +107,7 @@ type prollyConflictRowIter struct {
itr prolly.ConflictArtifactIter
tblName string
vrw types.ValueReadWriter
ns tree.NodeStore
ourRows prolly.Map
keyless bool
@@ -157,6 +159,7 @@ func newProllyConflictRowIter(ctx *sql.Context, ct ProllyConflictsTable) (*proll
itr: itr,
tblName: ct.tblName,
vrw: ct.tbl.ValueReadWriter(),
ns: ct.tbl.NodeStore(),
ourRows: ourRows,
keyless: keyless,
kd: kd,
@@ -325,7 +328,7 @@ func (itr *prollyConflictRowIter) nextConflictVals(ctx *sql.Context) (c conf, er
// the currently loaded maps. |baseHash| and |theirHash| are table hashes.
func (itr *prollyConflictRowIter) loadTableMaps(ctx context.Context, baseHash, theirHash hash.Hash) error {
if itr.baseHash.Compare(baseHash) != 0 {
rv, err := doltdb.LoadRootValueFromRootIshAddr(ctx, itr.vrw, baseHash)
rv, err := doltdb.LoadRootValueFromRootIshAddr(ctx, itr.vrw, itr.ns, baseHash)
if err != nil {
return err
}
@@ -346,7 +349,7 @@ func (itr *prollyConflictRowIter) loadTableMaps(ctx context.Context, baseHash, t
}
if itr.theirHash.Compare(theirHash) != 0 {
rv, err := doltdb.LoadRootValueFromRootIshAddr(ctx, itr.vrw, theirHash)
rv, err := doltdb.LoadRootValueFromRootIshAddr(ctx, itr.vrw, itr.ns, theirHash)
if err != nil {
return err
}
go/libraries/doltcore/sqle/dtables/diff_table.go
+1 -1
View File
@@ -229,7 +229,7 @@ func tableData(ctx *sql.Context, tbl *doltdb.Table, ddb *doltdb.DoltDB) (durable
var err error
if tbl == nil {
data, err = durable.NewEmptyIndex(ctx, ddb.ValueReadWriter(), schema.EmptySchema)
data, err = durable.NewEmptyIndex(ctx, ddb.ValueReadWriter(), ddb.NodeStore(), schema.EmptySchema)
if err != nil {
return nil, nil, err
}
go/libraries/doltcore/sqle/tables.go
+2 -2
View File
@@ -657,7 +657,7 @@ func (t *WritableDoltTable) Truncate(ctx *sql.Context) (int, error) {
// truncate returns an empty copy of the table given by setting the rows and indexes to empty. The schema can be
// updated at the same time.
func truncate(ctx *sql.Context, table *doltdb.Table, sch schema.Schema) (*doltdb.Table, error) {
empty, err := durable.NewEmptyIndex(ctx, table.ValueReadWriter(), sch)
empty, err := durable.NewEmptyIndex(ctx, table.ValueReadWriter(), table.NodeStore(), sch)
if err != nil {
return nil, err
}
@@ -675,7 +675,7 @@ func truncate(ctx *sql.Context, table *doltdb.Table, sch schema.Schema) (*doltdb
}
// truncate table resets auto-increment value
return doltdb.NewTable(ctx, table.ValueReadWriter(), sch, empty, idxSet, nil)
return doltdb.NewTable(ctx, table.ValueReadWriter(), table.NodeStore(), sch, empty, idxSet, nil)
}
// Updater implements sql.UpdatableTable
go/libraries/doltcore/sqle/temp_table.go
+4 -3
View File
@@ -84,14 +84,15 @@ func NewTempTable(
return nil, err
}
vrw := ddb.ValueReadWriter()
ns := ddb.NodeStore()
idx, err := durable.NewEmptyIndex(ctx, vrw, sch)
idx, err := durable.NewEmptyIndex(ctx, vrw, ns, sch)
if err != nil {
return nil, err
}
set := durable.NewIndexSet(ctx, vrw)
set := durable.NewIndexSet(ctx, vrw, ns)
tbl, err := doltdb.NewTable(ctx, ddb.ValueReadWriter(), sch, idx, set, nil)
tbl, err := doltdb.NewTable(ctx, vrw, ns, sch, idx, set, nil)
if err != nil {
return nil, err
}
go/libraries/doltcore/sqle/testdata.go
+1 -1
View File
@@ -537,7 +537,7 @@ func UpdateTables(t *testing.T, ctx context.Context, root *doltdb.RootValue, tbl
require.NoError(t, err)
}
tbl, err = doltdb.NewNomsTable(ctx, root.VRW(), sch, rowData, indexData, nil)
tbl, err = doltdb.NewNomsTable(ctx, root.VRW(), root.NodeStore(), sch, rowData, indexData, nil)
require.NoError(t, err)
root, err = root.PutTable(ctx, tblName, tbl)
go/libraries/doltcore/table/editor/creation/index.go
+7 -6
View File
@@ -29,6 +29,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
@@ -153,7 +154,7 @@ func BuildSecondaryIndex(ctx context.Context, tbl *doltdb.Table, idx schema.Inde
return nil, err
}
primary := durable.ProllyMapFromIndex(m)
return BuildSecondaryProllyIndex(ctx, tbl.ValueReadWriter(), sch, idx, primary)
return BuildSecondaryProllyIndex(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), sch, idx, primary)
default:
return nil, fmt.Errorf("unknown NomsBinFormat")
@@ -162,15 +163,15 @@ func BuildSecondaryIndex(ctx context.Context, tbl *doltdb.Table, idx schema.Inde
// BuildSecondaryProllyIndex builds secondary index data for the given primary
// index row data |primary|. |sch| is the current schema of the table.
func BuildSecondaryProllyIndex(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema, idx schema.Index, primary prolly.Map) (durable.Index, error) {
func BuildSecondaryProllyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, idx schema.Index, primary prolly.Map) (durable.Index, error) {
if idx.IsUnique() {
kd := shim.KeyDescriptorFromSchema(idx.Schema())
return BuildUniqueProllyIndex(ctx, vrw, sch, idx, primary, func(ctx context.Context, existingKey, newKey val.Tuple) error {
return BuildUniqueProllyIndex(ctx, vrw, ns, sch, idx, primary, func(ctx context.Context, existingKey, newKey val.Tuple) error {
return sql.ErrDuplicateEntry.Wrap(&prollyUniqueKeyErr{k: newKey, kd: kd, IndexName: idx.Name()}, idx.Name())
})
}
empty, err := durable.NewEmptyIndex(ctx, vrw, idx.Schema())
empty, err := durable.NewEmptyIndex(ctx, vrw, ns, idx.Schema())
if err != nil {
return nil, err
}
@@ -233,8 +234,8 @@ type DupEntryCb func(ctx context.Context, existingKey, newKey val.Tuple) error
// BuildUniqueProllyIndex builds a unique index based on the given |primary| row
// data. If any duplicate entries are found, they are passed to |cb|. If |cb|
// returns a non-nil error then the process is stopped.
func BuildUniqueProllyIndex(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema, idx schema.Index, primary prolly.Map, cb DupEntryCb) (durable.Index, error) {
empty, err := durable.NewEmptyIndex(ctx, vrw, idx.Schema())
func BuildUniqueProllyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, idx schema.Index, primary prolly.Map, cb DupEntryCb) (durable.Index, error) {
empty, err := durable.NewEmptyIndex(ctx, vrw, ns, idx.Schema())
if err != nil {
return nil, err
}
go/libraries/doltcore/table/editor/index_editor_test.go
+23 -22
View File
@@ -27,6 +27,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
@@ -52,7 +53,7 @@ var id3, _ = uuid.NewRandom()
func TestIndexEditorConcurrency(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
colColl := schema.NewColCollection(
schema.NewColumn("pk", 0, types.IntKind, true),
@@ -147,7 +148,7 @@ func TestIndexEditorConcurrency(t *testing.T) {
func TestIndexEditorConcurrencyPostInsert(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
colColl := schema.NewColCollection(
schema.NewColumn("pk", 0, types.IntKind, true),
@@ -239,7 +240,7 @@ func TestIndexEditorConcurrencyPostInsert(t *testing.T) {
func TestIndexEditorUniqueMultipleNil(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
colColl := schema.NewColCollection(
schema.NewColumn("pk", 0, types.IntKind, true),
@@ -284,7 +285,7 @@ func TestIndexEditorUniqueMultipleNil(t *testing.T) {
func TestIndexEditorWriteAfterFlush(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
colColl := schema.NewColCollection(
schema.NewColumn("pk", 0, types.IntKind, true),
@@ -352,7 +353,7 @@ func TestIndexEditorWriteAfterFlush(t *testing.T) {
func TestIndexEditorUniqueErrorDoesntPersist(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
colColl := schema.NewColCollection(
schema.NewColumn("pk", 0, types.IntKind, true),
@@ -394,7 +395,7 @@ func TestIndexEditorUniqueErrorDoesntPersist(t *testing.T) {
}
func TestIndexRebuildingWithZeroIndexes(t *testing.T) {
_, vrw, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
_, vrw, ns, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
tSchema := createTestSchema(t)
_, err := tSchema.Indexes().RemoveIndex(testSchemaIndexName)
require.NoError(t, err)
@@ -402,7 +403,7 @@ func TestIndexRebuildingWithZeroIndexes(t *testing.T) {
require.NoError(t, err)
rowData, _ := createTestRowData(t, vrw, tSchema)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, tSchema, rowData)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, tSchema, rowData)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
@@ -416,7 +417,7 @@ func TestIndexRebuildingWithZeroIndexes(t *testing.T) {
}
func TestIndexRebuildingWithOneIndex(t *testing.T) {
_, vrw, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
_, vrw, ns, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
tSchema := createTestSchema(t)
_, err := tSchema.Indexes().RemoveIndex(testSchemaIndexAge)
require.NoError(t, err)
@@ -437,7 +438,7 @@ func TestIndexRebuildingWithOneIndex(t *testing.T) {
require.NoError(t, err)
}
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, tSchema, rowData)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, tSchema, rowData)
require.NoError(t, err)
var indexRows []row.Row
@@ -468,7 +469,7 @@ func TestIndexRebuildingWithOneIndex(t *testing.T) {
}
func TestIndexRebuildingWithTwoIndexes(t *testing.T) {
_, vrw, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
_, vrw, ns, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
tSchema := createTestSchema(t)
indexName := tSchema.Indexes().GetByName(testSchemaIndexName)
@@ -482,7 +483,7 @@ func TestIndexRebuildingWithTwoIndexes(t *testing.T) {
rowData, rows := createTestRowData(t, vrw, tSchema)
indexNameExpectedRows, indexAgeExpectedRows := rowsToIndexRows(t, rows, indexName, indexAge)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, tSchema, rowData)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, tSchema, rowData)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
@@ -592,7 +593,7 @@ func TestIndexRebuildingWithTwoIndexes(t *testing.T) {
}
func TestIndexRebuildingUniqueSuccessOneCol(t *testing.T) {
_, vrw, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
_, vrw, ns, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
colColl := schema.NewColCollection(
schema.NewColumn("pk1", 1, types.IntKind, true, schema.NotNullConstraint{}),
schema.NewColumn("v1", 2, types.IntKind, false),
@@ -605,7 +606,7 @@ func TestIndexRebuildingUniqueSuccessOneCol(t *testing.T) {
row.TaggedValues{1: types.Int(2), 2: types.Int(2), 3: types.Int(2)},
row.TaggedValues{1: types.Int(3), 2: types.Int(3), 3: types.Int(3)},
)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, sch, rowData)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, sch, rowData)
require.NoError(t, err)
index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2}, schema.IndexProperties{IsUnique: true, Comment: ""})
@@ -622,7 +623,7 @@ func TestIndexRebuildingUniqueSuccessOneCol(t *testing.T) {
}
func TestIndexRebuildingUniqueSuccessTwoCol(t *testing.T) {
_, vrw, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
_, vrw, ns, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
colColl := schema.NewColCollection(
schema.NewColumn("pk1", 1, types.IntKind, true, schema.NotNullConstraint{}),
schema.NewColumn("v1", 2, types.IntKind, false),
@@ -635,7 +636,7 @@ func TestIndexRebuildingUniqueSuccessTwoCol(t *testing.T) {
row.TaggedValues{1: types.Int(2), 2: types.Int(1), 3: types.Int(2)},
row.TaggedValues{1: types.Int(3), 2: types.Int(2), 3: types.Int(2)},
)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, sch, rowData)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, sch, rowData)
require.NoError(t, err)
index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2, 3}, schema.IndexProperties{IsUnique: true, Comment: ""})
@@ -652,7 +653,7 @@ func TestIndexRebuildingUniqueSuccessTwoCol(t *testing.T) {
}
func TestIndexRebuildingUniqueFailOneCol(t *testing.T) {
_, vrw, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
_, vrw, ns, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
colColl := schema.NewColCollection(
schema.NewColumn("pk1", 1, types.IntKind, true, schema.NotNullConstraint{}),
schema.NewColumn("v1", 2, types.IntKind, false),
@@ -665,7 +666,7 @@ func TestIndexRebuildingUniqueFailOneCol(t *testing.T) {
row.TaggedValues{1: types.Int(2), 2: types.Int(2), 3: types.Int(2)},
row.TaggedValues{1: types.Int(3), 2: types.Int(2), 3: types.Int(3)},
)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, sch, rowData)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, sch, rowData)
require.NoError(t, err)
index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2}, schema.IndexProperties{IsUnique: true, Comment: ""})
@@ -682,7 +683,7 @@ func TestIndexRebuildingUniqueFailOneCol(t *testing.T) {
}
func TestIndexRebuildingUniqueFailTwoCol(t *testing.T) {
_, vrw, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
_, vrw, ns, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
colColl := schema.NewColCollection(
schema.NewColumn("pk1", 1, types.IntKind, true, schema.NotNullConstraint{}),
schema.NewColumn("v1", 2, types.IntKind, false),
@@ -696,7 +697,7 @@ func TestIndexRebuildingUniqueFailTwoCol(t *testing.T) {
row.TaggedValues{1: types.Int(3), 2: types.Int(2), 3: types.Int(2)},
row.TaggedValues{1: types.Int(4), 2: types.Int(1), 3: types.Int(2)},
)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, sch, rowData)
originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, sch, rowData)
require.NoError(t, err)
index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2, 3}, schema.IndexProperties{IsUnique: true, Comment: ""})
@@ -717,7 +718,7 @@ func TestIndexEditorCapacityExceeded(t *testing.T) {
// In the event that we reach the iea capacity on Undo, we need to verify that all code paths fail and remain failing
ctx := context.Background()
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(ctx, format, nil, nil)
_, vrw, _, err := dbfactory.MemFactory{}.CreateDB(ctx, format, nil, nil)
require.NoError(t, err)
colColl := schema.NewColCollection(
schema.NewColumn("pk", 0, types.IntKind, true),
@@ -830,8 +831,8 @@ func createTestSchema(t *testing.T) schema.Schema {
return sch
}
func createTableWithoutIndexRebuilding(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema, rowData types.Map) (*doltdb.Table, error) {
return doltdb.NewNomsTable(ctx, vrw, sch, rowData, nil, nil)
func createTableWithoutIndexRebuilding(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rowData types.Map) (*doltdb.Table, error) {
return doltdb.NewNomsTable(ctx, vrw, ns, sch, rowData, nil, nil)
}
func rowsToIndexRows(t *testing.T, rows []row.Row, indexName schema.Index, indexAge schema.Index) (indexNameExpectedRows []row.Row, indexAgeExpectedRows []row.Row) {
go/libraries/doltcore/table/editor/keyless_table_editor_test.go
+12 -12
View File
@@ -32,7 +32,7 @@ import (
func TestKeylessTableEditorConcurrency(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
colColl := schema.NewColCollection(
schema.NewColumn("v0", 0, types.IntKind, false),
@@ -42,7 +42,7 @@ func TestKeylessTableEditorConcurrency(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(context.Background(), vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(context.Background(), vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(context.Background(), vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
@@ -140,7 +140,7 @@ func TestKeylessTableEditorConcurrency(t *testing.T) {
func TestKeylessTableEditorConcurrencyPostInsert(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
colColl := schema.NewColCollection(
schema.NewColumn("v0", 0, types.IntKind, false),
@@ -150,7 +150,7 @@ func TestKeylessTableEditorConcurrencyPostInsert(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(context.Background(), vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(context.Background(), vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(context.Background(), vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
@@ -247,7 +247,7 @@ func TestKeylessTableEditorConcurrencyPostInsert(t *testing.T) {
func TestKeylessTableEditorWriteAfterFlush(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
colColl := schema.NewColCollection(
schema.NewColumn("v0", 0, types.IntKind, false),
@@ -257,7 +257,7 @@ func TestKeylessTableEditorWriteAfterFlush(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(context.Background(), vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(context.Background(), vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(context.Background(), vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
@@ -328,7 +328,7 @@ func TestKeylessTableEditorWriteAfterFlush(t *testing.T) {
func TestKeylessTableEditorDuplicateKeyHandling(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
colColl := schema.NewColCollection(
schema.NewColumn("v0", 0, types.IntKind, false),
@@ -338,7 +338,7 @@ func TestKeylessTableEditorDuplicateKeyHandling(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(context.Background(), vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(context.Background(), vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(context.Background(), vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
@@ -418,7 +418,7 @@ func TestKeylessTableEditorDuplicateKeyHandling(t *testing.T) {
func TestKeylessTableEditorMultipleIndexErrorHandling(t *testing.T) {
ctx := context.Background()
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(ctx, format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(ctx, format, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
colColl := schema.NewColCollection(
@@ -437,7 +437,7 @@ func TestKeylessTableEditorMultipleIndexErrorHandling(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(ctx, vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(ctx, vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(ctx, vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
table, err = RebuildAllIndexes(ctx, table, opts)
require.NoError(t, err)
@@ -575,7 +575,7 @@ func TestKeylessTableEditorMultipleIndexErrorHandling(t *testing.T) {
func TestKeylessTableEditorIndexCardinality(t *testing.T) {
ctx := context.Background()
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(ctx, format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(ctx, format, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
colColl := schema.NewColCollection(
@@ -590,7 +590,7 @@ func TestKeylessTableEditorIndexCardinality(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(ctx, vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(ctx, vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(ctx, vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
table, err = RebuildAllIndexes(ctx, table, opts)
require.NoError(t, err)
go/libraries/doltcore/table/editor/pk_table_editor_test.go
+10 -10
View File
@@ -42,7 +42,7 @@ const (
func TestTableEditorConcurrency(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
colColl := schema.NewColCollection(
@@ -53,7 +53,7 @@ func TestTableEditorConcurrency(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(context.Background(), vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(context.Background(), vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(context.Background(), vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
for i := 0; i < tableEditorConcurrencyIterations; i++ {
@@ -137,7 +137,7 @@ func TestTableEditorConcurrency(t *testing.T) {
func TestTableEditorConcurrencyPostInsert(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
colColl := schema.NewColCollection(
@@ -148,7 +148,7 @@ func TestTableEditorConcurrencyPostInsert(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(context.Background(), vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(context.Background(), vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(context.Background(), vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
tableEditor, err := newPkTableEditor(context.Background(), table, tableSch, tableName, opts)
@@ -230,7 +230,7 @@ func TestTableEditorConcurrencyPostInsert(t *testing.T) {
func TestTableEditorWriteAfterFlush(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
colColl := schema.NewColCollection(
@@ -241,7 +241,7 @@ func TestTableEditorWriteAfterFlush(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(context.Background(), vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(context.Background(), vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(context.Background(), vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
tableEditor, err := newPkTableEditor(context.Background(), table, tableSch, tableName, opts)
@@ -300,7 +300,7 @@ func TestTableEditorWriteAfterFlush(t *testing.T) {
func TestTableEditorDuplicateKeyHandling(t *testing.T) {
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
colColl := schema.NewColCollection(
@@ -311,7 +311,7 @@ func TestTableEditorDuplicateKeyHandling(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(context.Background(), vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(context.Background(), vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(context.Background(), vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
tableEditor, err := newPkTableEditor(context.Background(), table, tableSch, tableName, opts)
@@ -379,7 +379,7 @@ func TestTableEditorDuplicateKeyHandling(t *testing.T) {
func TestTableEditorMultipleIndexErrorHandling(t *testing.T) {
ctx := context.Background()
format := types.Format_Default
_, vrw, err := dbfactory.MemFactory{}.CreateDB(ctx, format, nil, nil)
_, vrw, ns, err := dbfactory.MemFactory{}.CreateDB(ctx, format, nil, nil)
require.NoError(t, err)
opts := TestEditorOptions(vrw)
colColl := schema.NewColCollection(
@@ -398,7 +398,7 @@ func TestTableEditorMultipleIndexErrorHandling(t *testing.T) {
require.NoError(t, err)
emptyMap, err := types.NewMap(ctx, vrw)
require.NoError(t, err)
table, err := doltdb.NewNomsTable(ctx, vrw, tableSch, emptyMap, nil, nil)
table, err := doltdb.NewNomsTable(ctx, vrw, ns, tableSch, emptyMap, nil, nil)
require.NoError(t, err)
table, err = RebuildAllIndexes(ctx, table, opts)
require.NoError(t, err)
go/libraries/doltcore/table/keyless_reader_test.go
+3 -2
View File
@@ -115,6 +115,7 @@ func TestKeylessTableReader(t *testing.T) {
dEnv := dtu.CreateTestEnv()
ctx := context.Background()
vrw := dEnv.DoltDB.ValueReadWriter()
ns := dEnv.DoltDB.NodeStore()
compareRows := func(t *testing.T, expected []sql.Row, rdr table.SqlTableReader) {
for _, exp := range expected {
@@ -130,7 +131,7 @@ func TestKeylessTableReader(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
rowMap := makeBag(vrw, sch, test.rows...)
tbl, err := doltdb.NewNomsTable(ctx, vrw, sch, rowMap, nil, nil)
tbl, err := doltdb.NewNomsTable(ctx, vrw, ns, sch, rowMap, nil, nil)
require.NoError(t, err)
rdr, err := table.NewTableReader(ctx, tbl)
require.NoError(t, err)
@@ -138,7 +139,7 @@ func TestKeylessTableReader(t *testing.T) {
})
t.Run(test.name+"_buffered", func(t *testing.T) {
rowMap := makeBag(vrw, sch, test.rows...)
tbl, err := doltdb.NewNomsTable(ctx, vrw, sch, rowMap, nil, nil)
tbl, err := doltdb.NewNomsTable(ctx, vrw, ns, sch, rowMap, nil, nil)
require.NoError(t, err)
rdr, err := table.NewBufferedTableReader(ctx, tbl)
require.NoError(t, err)
go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go
+2 -2
View File
@@ -93,11 +93,11 @@ func TestEndToEnd(t *testing.T) {
empty, err := types.NewMap(ctx, root.VRW())
require.NoError(t, err)
indexes := durable.NewIndexSet(ctx, root.VRW())
indexes := durable.NewIndexSet(ctx, root.VRW(), root.NodeStore())
indexes, err = indexes.PutNomsIndex(ctx, dtestutils.IndexName, empty)
require.NoError(t, err)
tbl, err := doltdb.NewNomsTable(ctx, root.VRW(), tt.sch, empty, indexes, nil)
tbl, err := doltdb.NewNomsTable(ctx, root.VRW(), root.NodeStore(), tt.sch, empty, indexes, nil)
require.NoError(t, err)
root, err = root.PutTable(ctx, tableName, tbl)
require.NoError(t, err)