From b5716180fc7fda126ba8fd18a5f54f662b3b034b Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Wed, 20 Jul 2022 14:43:14 -0700 Subject: [PATCH 01/15] remove empty error return value from Commit.NumParents --- go/cmd/dolt/commands/log.go | 8 +------- go/libraries/doltcore/doltdb/commit.go | 11 ++++------- go/libraries/doltcore/doltdb/doltdb.go | 5 +---- go/libraries/doltcore/doltdb/doltdb_test.go | 2 +- go/libraries/doltcore/rebase/rebase.go | 9 ++------- 5 files changed, 9 insertions(+), 26 deletions(-) diff --git a/go/cmd/dolt/commands/log.go b/go/cmd/dolt/commands/log.go index 8f278cbc69..b925cb8045 100644 --- a/go/cmd/dolt/commands/log.go +++ b/go/cmd/dolt/commands/log.go @@ -224,13 +224,7 @@ func logCommits(ctx context.Context, dEnv *env.DoltEnv, cs *doltdb.CommitSpec, o } matchFunc := func(commit *doltdb.Commit) (bool, error) { - numParents, err := commit.NumParents() - - if err != nil { - return false, err - } - - return numParents >= opts.minParents, nil + return commit.NumParents() >= opts.minParents, nil } commits, err := commitwalk.GetTopNTopoOrderedCommitsMatching(ctx, dEnv.DoltDB, h, opts.numLines, matchFunc) diff --git a/go/libraries/doltcore/doltdb/commit.go b/go/libraries/doltcore/doltdb/commit.go index 8133c13765..df67e05a75 100644 --- a/go/libraries/doltcore/doltdb/commit.go +++ b/go/libraries/doltcore/doltdb/commit.go @@ -80,8 +80,8 @@ func (c *Commit) ParentHashes(ctx context.Context) ([]hash.Hash, error) { } // NumParents gets the number of parents a commit has. -func (c *Commit) NumParents() (int, error) { - return len(c.parents), nil +func (c *Commit) NumParents() int { + return len(c.parents) } func (c *Commit) Height() (uint64, error) { @@ -180,14 +180,11 @@ func (c *Commit) GetAncestor(ctx context.Context, as *AncestorSpec) (*Commit, er instructions := as.Instructions for _, inst := range instructions { - n, err := cur.NumParents() - if err != nil { - return nil, err - } - if inst >= n { + if inst >= cur.NumParents() { return nil, ErrInvalidAncestorSpec } + var err error cur, err = cur.GetParent(ctx, inst) if err != nil { return nil, err diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index 55a0995390..912f737cff 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -633,10 +633,7 @@ func (ddb *DoltDB) ResolveParent(ctx context.Context, commit *Commit, parentIdx } func (ddb *DoltDB) ResolveAllParents(ctx context.Context, commit *Commit) ([]*Commit, error) { - num, err := commit.NumParents() - if err != nil { - return nil, err - } + num := commit.NumParents() resolved := make([]*Commit, num) for i := 0; i < num; i++ { parent, err := ddb.ResolveParent(ctx, commit, i) diff --git a/go/libraries/doltcore/doltdb/doltdb_test.go b/go/libraries/doltcore/doltdb/doltdb_test.go index 288ba99bff..fa4a0ed7c9 100644 --- a/go/libraries/doltcore/doltdb/doltdb_test.go +++ b/go/libraries/doltcore/doltdb/doltdb_test.go @@ -325,7 +325,7 @@ func TestLDNoms(t *testing.T) { assert.Equal(t, len(branches), 1) assert.Equal(t, branches[0].Ref.GetPath(), "master") - numParents, err := commit.NumParents() + numParents := commit.NumParents() assert.NoError(t, err) if numParents != 1 { diff --git a/go/libraries/doltcore/rebase/rebase.go b/go/libraries/doltcore/rebase/rebase.go index b9ba720c6e..f4289637e4 100644 --- a/go/libraries/doltcore/rebase/rebase.go +++ b/go/libraries/doltcore/rebase/rebase.go @@ -31,8 +31,7 @@ type NeedsRebaseFn func(ctx context.Context, cm *doltdb.Commit) (bool, error) // EntireHistory returns a |NeedsRebaseFn| that rebases the entire commit history. func EntireHistory() NeedsRebaseFn { return func(_ context.Context, cm *doltdb.Commit) (bool, error) { - n, err := cm.NumParents() - return n != 0, err + return cm.NumParents() != 0, nil } } @@ -54,11 +53,7 @@ func StopAtCommit(stopCommit *doltdb.Commit) NeedsRebaseFn { return false, nil } - n, err := cm.NumParents() - if err != nil { - return false, err - } - if n == 0 { + if cm.NumParents() == 0 { return false, fmt.Errorf("commit %s is missing from the commit history of at least one rebase head", sh) } From db7515ca1211803190ac06e7cddffb757c2670af Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Wed, 20 Jul 2022 15:23:49 -0700 Subject: [PATCH 02/15] sketch out migration --- go/libraries/doltcore/migrate/migrate.go | 102 +++++++++++++++++++++ go/libraries/doltcore/migrate/progress.go | 80 ++++++++++++++++ go/libraries/doltcore/migrate/transform.go | 29 ++++++ 3 files changed, 211 insertions(+) create mode 100644 go/libraries/doltcore/migrate/migrate.go create mode 100644 go/libraries/doltcore/migrate/progress.go create mode 100644 go/libraries/doltcore/migrate/transform.go diff --git a/go/libraries/doltcore/migrate/migrate.go b/go/libraries/doltcore/migrate/migrate.go new file mode 100644 index 0000000000..79c83e7a8e --- /dev/null +++ b/go/libraries/doltcore/migrate/migrate.go @@ -0,0 +1,102 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "context" + + "github.com/dolthub/dolt/go/store/hash" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/ref" +) + +func TraverseDAG(ctx context.Context, ddb *doltdb.DoltDB) (Progress, error) { + heads, err := ddb.GetHeadRefs(ctx) + if err != nil { + return nil, err + } + + prog := newProgress() + for i := range heads { + if err = TraverseHistory(ctx, heads[i], prog); err != nil { + return nil, err + } + } + return prog, nil +} + +func TraverseHistory(ctx context.Context, r ref.DoltRef, prog Progress) error { + return nil +} + +func TraverseTagHistory(ctx context.Context, tag *doltdb.Tag, prog Progress) error { + return nil +} + +func TraverseWorkingSetHistory(ctx context.Context, ws doltdb.WorkingSet, prog Progress) error { + return nil +} + +func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, prog Progress) error { + for { + ph, err := cm.ParentHashes(ctx) + if err != nil { + return err + } + + idx, err := firstAbsent(ctx, prog, ph) + if err != nil { + return err + } + if idx < 0 { + // parents for |cm| are done, migrate |cm| + if err = MigrateCommit(ctx, cm, prog); err != nil { + return err + } + // pop the stack, traverse upwards + cm, err = prog.Pop(ctx) + if err != nil { + return err + } + if cm == nil { + return nil // done + } + continue + } + + // push the stack, traverse downwards + if err = prog.Push(ctx, cm); err != nil { + return err + } + cm, err = cm.GetParent(ctx, idx) + if err != nil { + return err + } + } +} + +func firstAbsent(ctx context.Context, p Progress, addrs []hash.Hash) (int, error) { + for i := range addrs { + ok, err := p.Has(ctx, addrs[i]) + if err != nil { + return -1, err + } + if !ok { + return i, nil + } + } + return -1, nil +} diff --git a/go/libraries/doltcore/migrate/progress.go b/go/libraries/doltcore/migrate/progress.go new file mode 100644 index 0000000000..73a176b00d --- /dev/null +++ b/go/libraries/doltcore/migrate/progress.go @@ -0,0 +1,80 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "context" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/store/hash" +) + +type ChunkMapping interface { + Has(ctx context.Context, addr hash.Hash) (bool, error) + Get(ctx context.Context, addr hash.Hash) (hash.Hash, error) + Put(ctx context.Context, old, new hash.Hash) error +} + +type CommitStack interface { + Push(ctx context.Context, cm *doltdb.Commit) error + Pop(ctx context.Context) (*doltdb.Commit, error) +} + +type Progress interface { + ChunkMapping + CommitStack +} + +type memoryProgress struct { + stack []*doltdb.Commit + mapping map[hash.Hash]hash.Hash +} + +func newProgress() Progress { + return memoryProgress{ + stack: make([]*doltdb.Commit, 0, 128), + mapping: make(map[hash.Hash]hash.Hash, 128), + } +} + +func (mem memoryProgress) Has(ctx context.Context, addr hash.Hash) (ok bool, err error) { + _, ok = mem.mapping[addr] + return +} + +func (mem memoryProgress) Get(ctx context.Context, old hash.Hash) (new hash.Hash, err error) { + new = mem.mapping[old] + return +} + +func (mem memoryProgress) Put(ctx context.Context, old, new hash.Hash) (err error) { + mem.mapping[old] = new + return +} + +func (mem memoryProgress) Push(ctx context.Context, cm *doltdb.Commit) (err error) { + mem.stack = append(mem.stack, cm) + return +} + +func (mem memoryProgress) Pop(ctx context.Context) (cm *doltdb.Commit, err error) { + if len(mem.stack) == 0 { + return nil, nil + } + top := len(mem.stack) - 1 + cm = mem.stack[top] + mem.stack = mem.stack[:top] + return +} diff --git a/go/libraries/doltcore/migrate/transform.go b/go/libraries/doltcore/migrate/transform.go new file mode 100644 index 0000000000..50f874f9e5 --- /dev/null +++ b/go/libraries/doltcore/migrate/transform.go @@ -0,0 +1,29 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "context" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" +) + +func MigrateCommit(ctx context.Context, old *doltdb.Commit, prog Progress) error { + return nil +} + +func MigrateRoot(ctx context.Context, old *doltdb.RootValue) (*doltdb.RootValue, error) { + return old, nil +} From 226da75db397dd794366554ec4fdc1a8a14040b0 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Wed, 20 Jul 2022 17:01:51 -0700 Subject: [PATCH 03/15] added temp file method to utils/filesys --- go/libraries/utils/filesys/fs.go | 3 +++ go/libraries/utils/filesys/fs_test.go | 14 ++++++++++++++ go/libraries/utils/filesys/inmemfs.go | 9 +++++++++ go/libraries/utils/filesys/localfs.go | 4 ++++ 4 files changed, 30 insertions(+) diff --git a/go/libraries/utils/filesys/fs.go b/go/libraries/utils/filesys/fs.go index dc6cec5cf5..277084f266 100644 --- a/go/libraries/utils/filesys/fs.go +++ b/go/libraries/utils/filesys/fs.go @@ -71,6 +71,9 @@ type WritableFS interface { // MoveFile will move a file from the srcPath in the filesystem to the destPath MoveFile(srcPath, destPath string) error + + // TempDir returns the path of a new temporary directory. + TempDir() string } // FSIterCB specifies the signature of the function that will be called for every item found while iterating. diff --git a/go/libraries/utils/filesys/fs_test.go b/go/libraries/utils/filesys/fs_test.go index 9c605ef9e7..c3fec178fe 100644 --- a/go/libraries/utils/filesys/fs_test.go +++ b/go/libraries/utils/filesys/fs_test.go @@ -103,6 +103,20 @@ func TestFilesystems(t *testing.T) { dataRead, err = fs.ReadFile(movedFilePath) require.NoError(t, err) require.Equal(t, dataRead, data) + + tmp := fs.TempDir() + require.NotEmpty(t, tmp) + fp2 := filepath.Join(tmp, "data.txt") + wrc, err := fs.OpenForWrite(fp2, os.ModePerm) + require.NoError(t, err) + require.NoError(t, wrc.Close()) + + // Test writing/reading random data to tmp file + err = fs.WriteFile(fp2, data) + require.NoError(t, err) + dataRead, err = fs.ReadFile(fp2) + require.NoError(t, err) + require.Equal(t, dataRead, data) }) } } diff --git a/go/libraries/utils/filesys/inmemfs.go b/go/libraries/utils/filesys/inmemfs.go index 9a4a2fea5a..308ab283d6 100644 --- a/go/libraries/utils/filesys/inmemfs.go +++ b/go/libraries/utils/filesys/inmemfs.go @@ -16,8 +16,10 @@ package filesys import ( "bytes" + "encoding/base32" "errors" "io" + "math/rand" "os" "path/filepath" "strings" @@ -559,6 +561,13 @@ func (fs *InMemFS) LastModified(path string) (t time.Time, exists bool) { return time.Time{}, false } +func (fs *InMemFS) TempDir() string { + buf := make([]byte, 16) + rand.Read(buf) + s := base32.HexEncoding.EncodeToString(buf) + return "/var/folders/gc/" + s + "/T/" +} + func (fs *InMemFS) pathToNative(path string) string { if len(path) >= 1 { if path[0] == '.' { diff --git a/go/libraries/utils/filesys/localfs.go b/go/libraries/utils/filesys/localfs.go index 4869ff1472..4e8840e0af 100644 --- a/go/libraries/utils/filesys/localfs.go +++ b/go/libraries/utils/filesys/localfs.go @@ -318,3 +318,7 @@ func (fs *localFS) LastModified(path string) (t time.Time, exists bool) { return stat.ModTime(), true } + +func (fs *localFS) TempDir() string { + return os.TempDir() +} From c95153ca6cb30806ecc4860344f038e47e99f4b5 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Thu, 21 Jul 2022 14:44:44 -0700 Subject: [PATCH 04/15] added temp migration environment --- go/cmd/dolt/commands/migrate.go | 24 ++- go/libraries/doltcore/migrate/environment.go | 153 ++++++++++++++++++ go/libraries/doltcore/migrate/transform.go | 4 +- .../migrate/{migrate.go => traverse.go} | 26 +-- go/libraries/utils/filesys/fs.go | 15 ++ go/libraries/utils/filesys/inmemfs.go | 38 +++++ 6 files changed, 243 insertions(+), 17 deletions(-) create mode 100644 go/libraries/doltcore/migrate/environment.go rename go/libraries/doltcore/migrate/{migrate.go => traverse.go} (76%) diff --git a/go/cmd/dolt/commands/migrate.go b/go/cmd/dolt/commands/migrate.go index 81edda9864..fe2ac442d3 100644 --- a/go/cmd/dolt/commands/migrate.go +++ b/go/cmd/dolt/commands/migrate.go @@ -20,8 +20,10 @@ import ( "github.com/fatih/color" "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/cmd/dolt/errhand" eventsapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi/v1alpha1" "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/migrate" "github.com/dolthub/dolt/go/libraries/utils/argparser" ) @@ -36,7 +38,7 @@ const ( var migrateDocs = cli.CommandDocumentationContent{ ShortDesc: "Executes a database migration to use the latest Dolt data format.", LongDesc: `Migrate is a multi-purpose command to update the data format of a Dolt database. Over time, development -on Dolt requires changes to the on-disk data format. These changes are necessary to improve Database performance amd +on Dolt requires changes to the on-disk data format. These changes are necessary to improve Database performance and correctness. Migrating to the latest format is therefore necessary for compatibility with the latest Dolt clients, and to take advantage of the newly released Dolt features.`, @@ -76,7 +78,7 @@ func (cmd MigrateCmd) EventType() eventsapi.ClientEventType { // Exec executes the command func (cmd MigrateCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv) int { ap := cmd.ArgParser() - help, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, migrateDocs, ap)) + help, usage := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, migrateDocs, ap)) apr := cli.ParseArgsOrDie(ap, args, help) if apr.Contains(migratePushFlag) && apr.Contains(migratePullFlag) { @@ -84,5 +86,23 @@ func (cmd MigrateCmd) Exec(ctx context.Context, commandStr string, args []string return 1 } + if err := MigrateDatabase(ctx, dEnv); err != nil { + verr := errhand.BuildDError("migration failed").AddCause(err).Build() + return HandleVErrAndExitCode(verr, usage) + } return 0 } + +func MigrateDatabase(ctx context.Context, dEnv *env.DoltEnv) error { + menv, err := migrate.NewEnvironment(ctx, dEnv) + if err != nil { + return err + } + + err = migrate.TraverseDAG(ctx, menv.Migration.DoltDB, menv.Existing.DoltDB) + if err != nil { + return err + } + + return migrate.SwapChunkstores(ctx, menv.Migration, "") +} diff --git a/go/libraries/doltcore/migrate/environment.go b/go/libraries/doltcore/migrate/environment.go new file mode 100644 index 0000000000..a25ccdc2a2 --- /dev/null +++ b/go/libraries/doltcore/migrate/environment.go @@ -0,0 +1,153 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/utils/earl" + "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/store/types" + + "github.com/dolthub/dolt/go/libraries/doltcore/env" +) + +const ( + doltDir = dbfactory.DoltDir + nomsDir = dbfactory.DataDir +) + +var targetFormat = types.Format_DOLT_DEV + +type Environment struct { + Migration *env.DoltEnv + Existing *env.DoltEnv +} + +func NewEnvironment(ctx context.Context, existing *env.DoltEnv) (Environment, error) { + mfs, err := getMigrateFS(existing.FS) + if err != nil { + return Environment{}, err + } + + if err = InitMigrationDB(ctx, existing.FS, mfs); err != nil { + return Environment{}, err + } + + mdb, err := doltdb.LoadDoltDB(ctx, targetFormat, doltdb.LocalDirDoltDB, mfs) + if err != nil { + return Environment{}, err + } + + config, err := env.LoadDoltCliConfig(env.GetCurrentUserHomeDir, mfs) + if err != nil { + return Environment{}, err + } + + migration := &env.DoltEnv{ + Version: existing.Version, + Config: config, + RepoState: existing.RepoState, + DoltDB: mdb, + FS: mfs, + //urlStr: urlStr, + //hdp: hdp, + } + + return Environment{ + Migration: migration, + Existing: existing, + }, nil +} + +func SwapChunkstores(ctx context.Context, migration *env.DoltEnv, dest string) error { + return nil +} + +func InitMigrationDB(ctx context.Context, src, dest filesys.Filesys) (err error) { + base, err := src.Abs(".") + if err != nil { + return err + } + + ierr := src.Iter(doltDir, true, func(path string, size int64, isDir bool) (stop bool) { + if isDir { + err = dest.MkDirs(path) + stop = err != nil + return + } + if strings.Contains(path, nomsDir) { + return + } + + path, err = filepath.Rel(base, path) + if err != nil { + stop = true + return + } + + if err = filesys.CopyFile(path, path, src, dest); err != nil { + stop = true + return + } + return + }) + if ierr != nil { + return ierr + } + if err != nil { + return err + } + + dd, err := dest.Abs(filepath.Join(doltDir, nomsDir)) + if err != nil { + return err + } + if err = dest.MkDirs(dd); err != nil { + return err + } + + u, err := earl.Parse(dd) + if err != nil { + return err + } + + _, _, _, err = dbfactory.FileFactory{}.CreateDB(ctx, targetFormat, u, nil) + return +} + +func getMigrateFS(existing filesys.Filesys) (filesys.Filesys, error) { + uniq := fmt.Sprintf("dolt_migration_%d", time.Now().Unix()) + tmpPath := filepath.Join(existing.TempDir(), uniq) + if err := existing.MkDirs(tmpPath); err != nil { + return nil, err + } + + mfs, err := filesys.LocalFilesysWithWorkingDir(tmpPath) + if err != nil { + return nil, err + } + + if err = mfs.MkDirs(doltDir); err != nil { + return nil, err + } + return mfs, nil +} diff --git a/go/libraries/doltcore/migrate/transform.go b/go/libraries/doltcore/migrate/transform.go index 50f874f9e5..e6fecd272f 100644 --- a/go/libraries/doltcore/migrate/transform.go +++ b/go/libraries/doltcore/migrate/transform.go @@ -20,10 +20,10 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" ) -func MigrateCommit(ctx context.Context, old *doltdb.Commit, prog Progress) error { +func MigrateCommit(ctx context.Context, old *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { return nil } -func MigrateRoot(ctx context.Context, old *doltdb.RootValue) (*doltdb.RootValue, error) { +func MigrateRoot(ctx context.Context, old *doltdb.RootValue, new *doltdb.DoltDB) (*doltdb.RootValue, error) { return old, nil } diff --git a/go/libraries/doltcore/migrate/migrate.go b/go/libraries/doltcore/migrate/traverse.go similarity index 76% rename from go/libraries/doltcore/migrate/migrate.go rename to go/libraries/doltcore/migrate/traverse.go index 79c83e7a8e..fa814b9f42 100644 --- a/go/libraries/doltcore/migrate/migrate.go +++ b/go/libraries/doltcore/migrate/traverse.go @@ -23,34 +23,34 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/ref" ) -func TraverseDAG(ctx context.Context, ddb *doltdb.DoltDB) (Progress, error) { - heads, err := ddb.GetHeadRefs(ctx) +func TraverseDAG(ctx context.Context, old, new *doltdb.DoltDB) error { + heads, err := old.GetHeadRefs(ctx) if err != nil { - return nil, err + return err } prog := newProgress() for i := range heads { - if err = TraverseHistory(ctx, heads[i], prog); err != nil { - return nil, err + if err = TraverseRefHistory(ctx, heads[i], new, prog); err != nil { + return err } } - return prog, nil -} - -func TraverseHistory(ctx context.Context, r ref.DoltRef, prog Progress) error { return nil } -func TraverseTagHistory(ctx context.Context, tag *doltdb.Tag, prog Progress) error { +func TraverseRefHistory(ctx context.Context, r ref.DoltRef, new *doltdb.DoltDB, prog Progress) error { return nil } -func TraverseWorkingSetHistory(ctx context.Context, ws doltdb.WorkingSet, prog Progress) error { +func TraverseTagHistory(ctx context.Context, tag *doltdb.Tag, new *doltdb.DoltDB, prog Progress) error { return nil } -func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, prog Progress) error { +func TraverseWorkingSetHistory(ctx context.Context, ws doltdb.WorkingSet, new *doltdb.DoltDB, prog Progress) error { + return nil +} + +func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { for { ph, err := cm.ParentHashes(ctx) if err != nil { @@ -63,7 +63,7 @@ func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, prog Progress } if idx < 0 { // parents for |cm| are done, migrate |cm| - if err = MigrateCommit(ctx, cm, prog); err != nil { + if err = MigrateCommit(ctx, cm, new, prog); err != nil { return err } // pop the stack, traverse upwards diff --git a/go/libraries/utils/filesys/fs.go b/go/libraries/utils/filesys/fs.go index 277084f266..166c7ea377 100644 --- a/go/libraries/utils/filesys/fs.go +++ b/go/libraries/utils/filesys/fs.go @@ -114,3 +114,18 @@ func UnmarshalJSONFile(fs ReadableFS, path string, dest interface{}) error { return json.Unmarshal(data, dest) } + +func CopyFile(srcPath, destPath string, srcFS, destFS Filesys) (err error) { + rd, err := srcFS.OpenForRead(srcPath) + if err != nil { + return err + } + + wr, err := destFS.OpenForWrite(destPath, os.ModePerm) + if err != nil { + return err + } + + _, err = io.Copy(wr, rd) + return +} diff --git a/go/libraries/utils/filesys/inmemfs.go b/go/libraries/utils/filesys/inmemfs.go index 308ab283d6..f2111a8b68 100644 --- a/go/libraries/utils/filesys/inmemfs.go +++ b/go/libraries/utils/filesys/inmemfs.go @@ -537,6 +537,44 @@ func (fs *InMemFS) MoveFile(srcPath, destPath string) error { return os.ErrNotExist } +func (fs *InMemFS) CopyFile(srcPath, destPath string) error { + fs.rwLock.Lock() + defer fs.rwLock.Unlock() + + srcPath = fs.getAbsPath(srcPath) + destPath = fs.getAbsPath(destPath) + + if exists, destIsDir := fs.exists(destPath); exists && destIsDir { + return ErrIsDir + } + + if obj, ok := fs.objs[srcPath]; ok { + if obj.isDir() { + return ErrIsDir + } + + destDir := filepath.Dir(destPath) + destParentDir, err := fs.mkDirs(destDir) + if err != nil { + return err + } + + destData := make([]byte, len(obj.(*memFile).data)) + copy(destData, obj.(*memFile).data) + + now := InMemNowFunc() + destObj := &memFile{destPath, destData, destParentDir, now} + + fs.objs[destPath] = destObj + destParentDir.objs[destPath] = destObj + destParentDir.time = now + + return nil + } + + return os.ErrNotExist +} + // converts a path to an absolute path. If it's already an absolute path the input path will be returned unaltered func (fs *InMemFS) Abs(path string) (string, error) { path = fs.pathToNative(path) From 1c49de8032c2a2992793e3a6f5ff91f25304c450 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 22 Jul 2022 15:02:39 -0700 Subject: [PATCH 05/15] first pass at __DOLT_DEV__ migration --- go/cmd/dolt/commands/migrate.go | 7 +- go/libraries/doltcore/doltdb/doltdb.go | 23 ++- go/libraries/doltcore/doltdb/root_val.go | 4 + go/libraries/doltcore/migrate/environment.go | 45 ++++- go/libraries/doltcore/migrate/transform.go | 175 ++++++++++++++++++- go/libraries/doltcore/migrate/traverse.go | 103 ++++++++++- go/store/types/map.go | 2 +- 7 files changed, 330 insertions(+), 29 deletions(-) diff --git a/go/cmd/dolt/commands/migrate.go b/go/cmd/dolt/commands/migrate.go index fe2ac442d3..2712ebe198 100644 --- a/go/cmd/dolt/commands/migrate.go +++ b/go/cmd/dolt/commands/migrate.go @@ -98,8 +98,13 @@ func MigrateDatabase(ctx context.Context, dEnv *env.DoltEnv) error { if err != nil { return err } + p, err := menv.Migration.FS.Abs(".") + if err != nil { + return err + } + cli.Println("migrating database at tmp dir: ", p) - err = migrate.TraverseDAG(ctx, menv.Migration.DoltDB, menv.Existing.DoltDB) + err = migrate.TraverseDAG(ctx, menv.Existing.DoltDB, menv.Migration.DoltDB) if err != nil { return err } diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index 912f737cff..d29a4282bf 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -47,7 +47,7 @@ const ( ) const ( - creationBranch = "create" + CreationBranch = "create" defaultChunksPerTF = 256 * 1024 ) @@ -155,7 +155,7 @@ func (ddb *DoltDB) WriteEmptyRepoWithCommitTimeAndDefaultBranch( panic("Passed bad name or email. Both should be valid") } - ds, err := ddb.db.GetDataset(ctx, creationBranch) + ds, err := ddb.db.GetDataset(ctx, CreationBranch) if err != nil { return err @@ -181,7 +181,7 @@ func (ddb *DoltDB) WriteEmptyRepoWithCommitTimeAndDefaultBranch( commitOpts := datas.CommitOptions{Meta: cm} - cb := ref.NewInternalRef(creationBranch) + cb := ref.NewInternalRef(CreationBranch) ds, err = ddb.db.GetDataset(ctx, cb.String()) if err != nil { @@ -525,10 +525,13 @@ func (ddb *DoltDB) CommitWithParentCommits(ctx context.Context, valHash hash.Has parents = append(parents, addr) } } - commitOpts := datas.CommitOptions{Parents: parents, Meta: cm} - ds, err = ddb.db.GetDataset(ctx, dref.String()) + return ddb.CommitValue(ctx, dref, val, commitOpts) +} + +func (ddb *DoltDB) CommitValue(ctx context.Context, dref ref.DoltRef, val types.Value, commitOpts datas.CommitOptions) (*Commit, error) { + ds, err := ddb.db.GetDataset(ctx, dref.String()) if err != nil { return nil, err } @@ -573,9 +576,15 @@ func (ddb *DoltDB) CommitDanglingWithParentCommits(ctx context.Context, valHash } parents = append(parents, addr) } - commitOpts := datas.CommitOptions{Parents: parents, Meta: cm} - dcommit, err := datas.NewCommitForValue(ctx, datas.ChunkStoreFromDatabase(ddb.db), ddb.vrw, ddb.ns, val, commitOpts) + + return ddb.CommitDangling(ctx, val, commitOpts) +} + +func (ddb *DoltDB) CommitDangling(ctx context.Context, val types.Value, opts datas.CommitOptions) (*Commit, error) { + cs := datas.ChunkStoreFromDatabase(ddb.db) + + dcommit, err := datas.NewCommitForValue(ctx, cs, ddb.vrw, ddb.ns, val, opts) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/doltdb/root_val.go b/go/libraries/doltcore/doltdb/root_val.go index e42c3c286d..746e1e96bd 100644 --- a/go/libraries/doltcore/doltdb/root_val.go +++ b/go/libraries/doltcore/doltdb/root_val.go @@ -768,6 +768,10 @@ func (root *RootValue) withStorage(st rvStorage) *RootValue { return &RootValue{root.vrw, root.ns, st, nil} } +func HackNomsValuesFromRootValues(root *RootValue) types.Value { + return root.nomsValue() +} + func (root *RootValue) nomsValue() types.Value { return root.st.nomsValue() } diff --git a/go/libraries/doltcore/migrate/environment.go b/go/libraries/doltcore/migrate/environment.go index a25ccdc2a2..b88acf58db 100644 --- a/go/libraries/doltcore/migrate/environment.go +++ b/go/libraries/doltcore/migrate/environment.go @@ -23,16 +23,18 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/libraries/utils/earl" "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/types" - - "github.com/dolthub/dolt/go/libraries/doltcore/env" ) const ( - doltDir = dbfactory.DoltDir - nomsDir = dbfactory.DataDir + doltDir = dbfactory.DoltDir + nomsDir = dbfactory.DataDir + migrateBranch = "migrate" ) var targetFormat = types.Format_DOLT_DEV @@ -48,7 +50,7 @@ func NewEnvironment(ctx context.Context, existing *env.DoltEnv) (Environment, er return Environment{}, err } - if err = InitMigrationDB(ctx, existing.FS, mfs); err != nil { + if err = InitMigrationDB(ctx, existing.DoltDB, existing.FS, mfs); err != nil { return Environment{}, err } @@ -82,7 +84,7 @@ func SwapChunkstores(ctx context.Context, migration *env.DoltEnv, dest string) e return nil } -func InitMigrationDB(ctx context.Context, src, dest filesys.Filesys) (err error) { +func InitMigrationDB(ctx context.Context, existing *doltdb.DoltDB, src, dest filesys.Filesys) (err error) { base, err := src.Abs(".") if err != nil { return err @@ -130,8 +132,35 @@ func InitMigrationDB(ctx context.Context, src, dest filesys.Filesys) (err error) return err } - _, _, _, err = dbfactory.FileFactory{}.CreateDB(ctx, targetFormat, u, nil) - return + db, vrw, ns, err := dbfactory.FileFactory{}.CreateDB(ctx, targetFormat, u, nil) + if err != nil { + return err + } + + // migrate init commit + creation := ref.NewInternalRef(doltdb.CreationBranch) + init, err := existing.ResolveCommitRef(ctx, creation) + if err != nil { + return err + } + + meta, err := init.GetCommitMeta(ctx) + if err != nil { + return err + } + rv, err := doltdb.EmptyRootValue(ctx, vrw, ns) + if err != nil { + return err + } + nv := doltdb.HackNomsValuesFromRootValues(rv) + + ds, err := db.GetDataset(ctx, creation.String()) + if err != nil { + return err + } + + _, err = db.Commit(ctx, ds, nv, datas.CommitOptions{Meta: meta}) + return nil } func getMigrateFS(existing filesys.Filesys) (filesys.Filesys, error) { diff --git a/go/libraries/doltcore/migrate/transform.go b/go/libraries/doltcore/migrate/transform.go index e6fecd272f..254c9d3119 100644 --- a/go/libraries/doltcore/migrate/transform.go +++ b/go/libraries/doltcore/migrate/transform.go @@ -18,12 +18,179 @@ import ( "context" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/ref" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/store/datas" + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/types" ) -func MigrateCommit(ctx context.Context, old *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { - return nil +func MigrateCommit(ctx context.Context, r ref.DoltRef, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { + root, err := cm.GetRootValue(ctx) + if err != nil { + return err + } + + mRoot, err := MigrateRoot(ctx, root, new) + if err != nil { + return err + } + _, addr, err := new.WriteRootValue(ctx, mRoot) + if err != nil { + return err + } + value, err := new.ValueReadWriter().ReadValue(ctx, addr) + if err != nil { + return err + } + + opts, err := MigrateCommitOptions(ctx, cm, prog) + if err != nil { + return err + } + + migratedCm, err := new.CommitValue(ctx, r, value, opts) + if err != nil { + return err + } + + // update progress + oldHash, err := cm.HashOf() + if err != nil { + return err + } + newHash, err := migratedCm.HashOf() + if err != nil { + return err + } + + return prog.Put(ctx, oldHash, newHash) } -func MigrateRoot(ctx context.Context, old *doltdb.RootValue, new *doltdb.DoltDB) (*doltdb.RootValue, error) { - return old, nil +func MigrateCommitOptions(ctx context.Context, oldCm *doltdb.Commit, prog Progress) (datas.CommitOptions, error) { + parents, err := oldCm.ParentHashes(ctx) + if err != nil { + return datas.CommitOptions{}, err + } + + for i := range parents { + migrated, err := prog.Get(ctx, parents[i]) + if err != nil { + return datas.CommitOptions{}, err + } + parents[i] = migrated + } + + meta, err := oldCm.GetCommitMeta(ctx) + if err != nil { + return datas.CommitOptions{}, err + } + + return datas.CommitOptions{ + Parents: parents, + Meta: meta, + }, nil +} + +func MigrateRoot(ctx context.Context, root *doltdb.RootValue, new *doltdb.DoltDB) (*doltdb.RootValue, error) { + migrated, err := doltdb.EmptyRootValue(ctx, new.ValueReadWriter(), new.NodeStore()) + if err != nil { + return nil, err + } + + fkc, err := root.GetForeignKeyCollection(ctx) + if err != nil { + return nil, err + } + + migrated, err = migrated.PutForeignKeyCollection(ctx, fkc) + if err != nil { + return nil, err + } + + err = root.IterTables(ctx, func(name string, tbl *doltdb.Table, _ schema.Schema) (bool, error) { + mtbl, err := MigrateTable(ctx, tbl, new) + if err != nil { + return true, err + } + + migrated, err = migrated.PutTable(ctx, name, mtbl) + if err != nil { + return true, err + } + return false, nil + }) + if err != nil { + return nil, err + } + + return migrated, nil +} + +func MigrateTable(ctx context.Context, table *doltdb.Table, new *doltdb.DoltDB) (*doltdb.Table, error) { + rows, err := table.GetRowData(ctx) + if err != nil { + return nil, err + } + + err = MigrateNomsMap(ctx, rows, table.ValueReadWriter(), new.ValueReadWriter()) + if err != nil { + return nil, err + } + + ai, err := table.GetAutoIncrementValue(ctx) + if err != nil { + return nil, err + } + autoInc := types.Uint(ai) + + sch, err := table.GetSchema(ctx) + if err != nil { + return nil, err + } + + oldSet, err := table.GetIndexSet(ctx) + if err != nil { + return nil, err + } + + newSet, err := MigrateIndexSet(ctx, sch, oldSet, table.ValueReadWriter(), new) + if err != nil { + return nil, err + } + + return doltdb.NewTable(ctx, new.ValueReadWriter(), new.NodeStore(), sch, rows, newSet, autoInc) +} + +func MigrateIndexSet(ctx context.Context, sch schema.Schema, oldSet durable.IndexSet, old types.ValueReadWriter, new *doltdb.DoltDB) (durable.IndexSet, error) { + newSet := durable.NewIndexSet(ctx, new.ValueReadWriter(), new.NodeStore()) + for _, def := range sch.Indexes().AllIndexes() { + idx, err := oldSet.GetIndex(ctx, sch, def.Name()) + if err != nil { + return nil, err + } + if err = MigrateNomsMap(ctx, idx, old, new.ValueReadWriter()); err != nil { + return nil, err + } + + newSet, err = newSet.PutIndex(ctx, def.Name(), idx) + if err != nil { + return nil, err + } + } + return newSet, nil +} + +func MigrateNomsMap(ctx context.Context, idx durable.Index, old, new types.ValueReadWriter) error { + m := durable.NomsMapFromIndex(idx) + _, _, err := types.VisitMapLevelOrder(m, func(h hash.Hash) (int64, error) { + v, err := old.ReadValue(ctx, h) + if err != nil { + return 0, err + } + _, err = new.WriteValue(ctx, v) + return 0, err + }) + return err } diff --git a/go/libraries/doltcore/migrate/traverse.go b/go/libraries/doltcore/migrate/traverse.go index fa814b9f42..6c6a403df3 100644 --- a/go/libraries/doltcore/migrate/traverse.go +++ b/go/libraries/doltcore/migrate/traverse.go @@ -16,6 +16,7 @@ package migrate import ( "context" + "fmt" "github.com/dolthub/dolt/go/store/hash" @@ -31,26 +32,112 @@ func TraverseDAG(ctx context.Context, old, new *doltdb.DoltDB) error { prog := newProgress() for i := range heads { - if err = TraverseRefHistory(ctx, heads[i], new, prog); err != nil { + if err = TraverseRefHistory(ctx, heads[i], old, new, prog); err != nil { return err } } + return nil } -func TraverseRefHistory(ctx context.Context, r ref.DoltRef, new *doltdb.DoltDB, prog Progress) error { +func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error { + // map init commits + o, err := old.ResolveCommitRef(ctx, ref.NewInternalRef(doltdb.CreationBranch)) + if err != nil { + return err + } + oh, err := o.HashOf() + if err != nil { + return err + } + n, err := new.ResolveCommitRef(ctx, ref.NewInternalRef(doltdb.CreationBranch)) + if err != nil { + return err + } + nh, err := n.HashOf() + if err != nil { + return err + } + if err = prog.Put(ctx, oh, nh); err != nil { + return err + } + + switch r.GetType() { + case ref.BranchRefType: + return TraverseBranchHistory(ctx, r.(ref.BranchRef), old, new, prog) + + case ref.TagRefType: + t, err := old.ResolveTag(ctx, r.(ref.TagRef)) + if err != nil { + return err + } + return TraverseTagHistory(ctx, t, old, new, prog) + + case ref.RemoteRefType: + return nil // todo(andy) + + case ref.WorkspaceRefType: + return nil // todo(andy) + + case ref.InternalRefType: + return nil // todo(andy)? + + default: + panic(fmt.Sprintf("unknown ref type %s", r.String())) + } +} + +func TraverseBranchHistory(ctx context.Context, r ref.BranchRef, old, new *doltdb.DoltDB, prog Progress) error { + cm, err := old.ResolveCommitRef(ctx, r) + if err != nil { + return err + } + if err = TraverseCommitHistory(ctx, r, cm, new, prog); err != nil { + return err + } + + oldHash, err := cm.HashOf() + if err != nil { + return err + } + newHash, err := prog.Get(ctx, oldHash) + if err != nil { + return err + } + if err = new.SetHead(ctx, r, newHash); err != nil { + return err + } + + wsRef, err := ref.WorkingSetRefForHead(r) + if err != nil { + return err + } + + ws, err := old.ResolveWorkingSet(ctx, wsRef) + if err != nil { + return err + } + return TraverseWorkingSetHistory(ctx, ws, old, new, prog) +} + +func TraverseTagHistory(ctx context.Context, tag *doltdb.Tag, old, new *doltdb.DoltDB, prog Progress) error { return nil } -func TraverseTagHistory(ctx context.Context, tag *doltdb.Tag, new *doltdb.DoltDB, prog Progress) error { +func TraverseWorkingSetHistory(ctx context.Context, ws *doltdb.WorkingSet, old, new *doltdb.DoltDB, prog Progress) error { return nil } -func TraverseWorkingSetHistory(ctx context.Context, ws doltdb.WorkingSet, new *doltdb.DoltDB, prog Progress) error { - return nil -} +func TraverseCommitHistory(ctx context.Context, r ref.DoltRef, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { + ch, err := cm.HashOf() + if err != nil { + return err + } + ok, err := prog.Has(ctx, ch) + if err != nil || ok { + return err + } -func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { for { ph, err := cm.ParentHashes(ctx) if err != nil { @@ -63,7 +150,7 @@ func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.D } if idx < 0 { // parents for |cm| are done, migrate |cm| - if err = MigrateCommit(ctx, cm, new, prog); err != nil { + if err = MigrateCommit(ctx, r, cm, new, prog); err != nil { return err } // pop the stack, traverse upwards diff --git a/go/store/types/map.go b/go/store/types/map.go index 5bb8369b7e..021317d4c1 100644 --- a/go/store/types/map.go +++ b/go/store/types/map.go @@ -627,7 +627,7 @@ func (m Map) HumanReadableString() string { } // VisitMapLevelOrder writes hashes of internal node chunks to a writer -// delimited with a newline character and returns the number or chunks written and the total number of +// delimited with a newline character and returns the number of chunks written and the total number of // bytes written or an error if encountered func VisitMapLevelOrder(m Map, cb func(h hash.Hash) (int64, error)) (int64, int64, error) { chunkCount := int64(0) From 237dbf8f9a642fa4c4eeabc212da2a34b3c21efb Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 22 Jul 2022 15:29:31 -0700 Subject: [PATCH 06/15] swapping chunk stores, manifests after migration (hope this works! ;-) --- go/cmd/dolt/commands/migrate.go | 2 +- go/libraries/doltcore/migrate/environment.go | 49 ++++++++++++++++++-- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/go/cmd/dolt/commands/migrate.go b/go/cmd/dolt/commands/migrate.go index 2712ebe198..ad6edce6fe 100644 --- a/go/cmd/dolt/commands/migrate.go +++ b/go/cmd/dolt/commands/migrate.go @@ -109,5 +109,5 @@ func MigrateDatabase(ctx context.Context, dEnv *env.DoltEnv) error { return err } - return migrate.SwapChunkstores(ctx, menv.Migration, "") + return migrate.SwapChunkStores(ctx, menv) } diff --git a/go/libraries/doltcore/migrate/environment.go b/go/libraries/doltcore/migrate/environment.go index b88acf58db..127d3d2bb3 100644 --- a/go/libraries/doltcore/migrate/environment.go +++ b/go/libraries/doltcore/migrate/environment.go @@ -32,9 +32,9 @@ import ( ) const ( - doltDir = dbfactory.DoltDir - nomsDir = dbfactory.DataDir - migrateBranch = "migrate" + doltDir = dbfactory.DoltDir + nomsDir = dbfactory.DataDir + manifestFile = "manifest" ) var targetFormat = types.Format_DOLT_DEV @@ -80,8 +80,47 @@ func NewEnvironment(ctx context.Context, existing *env.DoltEnv) (Environment, er }, nil } -func SwapChunkstores(ctx context.Context, migration *env.DoltEnv, dest string) error { - return nil +func SwapChunkStores(ctx context.Context, menv Environment) error { + src, dest := menv.Migration.FS, menv.Existing.FS + p := filepath.Join(doltDir, nomsDir) + + var cerr error + err := src.Iter(p, true, func(path string, size int64, isDir bool) (stop bool) { + if strings.Contains(path, manifestFile) || isDir { + return + } + if cerr = filesys.CopyFile(path, path, src, dest); cerr != nil { + stop = true + } + return + }) + if err != nil { + return err + } + if cerr != nil { + return cerr + } + + return SwapManifests(ctx, src, dest) +} + +func SwapManifests(ctx context.Context, src, dest filesys.Filesys) (err error) { + // backup the current manifest + manifest := filepath.Join(doltDir, nomsDir, manifestFile) + bak := filepath.Join(doltDir, nomsDir, manifestFile+".bak") + if err = filesys.CopyFile(manifest, bak, dest, dest); err != nil { + return err + } + + // copy manifest to |dest| under temporary name + tmp := filepath.Join(doltDir, nomsDir, "temp-manifest") + if err = filesys.CopyFile(manifest, tmp, src, dest); err != nil { + return err + } + + // atomically swap the manifests + return dest.MoveFile(tmp, manifest) + // exit immediately! } func InitMigrationDB(ctx context.Context, existing *doltdb.DoltDB, src, dest filesys.Filesys) (err error) { From 05258b69323da6d4b99429321fc2c31ac7523561 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 25 Jul 2022 12:53:40 -0700 Subject: [PATCH 07/15] iterating on DOLT_DEV migration --- go/libraries/doltcore/migrate/environment.go | 154 +++++++++++++------ go/libraries/doltcore/migrate/progress.go | 19 ++- go/libraries/doltcore/migrate/transform.go | 10 +- go/libraries/doltcore/migrate/traverse.go | 55 ++++--- go/libraries/doltcore/migrate/validation.go | 48 ++++++ go/store/hash/hash.go | 15 ++ go/store/types/value_store.go | 3 +- 7 files changed, 224 insertions(+), 80 deletions(-) create mode 100644 go/libraries/doltcore/migrate/validation.go diff --git a/go/libraries/doltcore/migrate/environment.go b/go/libraries/doltcore/migrate/environment.go index 127d3d2bb3..264c0a17c0 100644 --- a/go/libraries/doltcore/migrate/environment.go +++ b/go/libraries/doltcore/migrate/environment.go @@ -16,6 +16,7 @@ package migrate import ( "context" + "errors" "fmt" "path/filepath" "strings" @@ -80,49 +81,6 @@ func NewEnvironment(ctx context.Context, existing *env.DoltEnv) (Environment, er }, nil } -func SwapChunkStores(ctx context.Context, menv Environment) error { - src, dest := menv.Migration.FS, menv.Existing.FS - p := filepath.Join(doltDir, nomsDir) - - var cerr error - err := src.Iter(p, true, func(path string, size int64, isDir bool) (stop bool) { - if strings.Contains(path, manifestFile) || isDir { - return - } - if cerr = filesys.CopyFile(path, path, src, dest); cerr != nil { - stop = true - } - return - }) - if err != nil { - return err - } - if cerr != nil { - return cerr - } - - return SwapManifests(ctx, src, dest) -} - -func SwapManifests(ctx context.Context, src, dest filesys.Filesys) (err error) { - // backup the current manifest - manifest := filepath.Join(doltDir, nomsDir, manifestFile) - bak := filepath.Join(doltDir, nomsDir, manifestFile+".bak") - if err = filesys.CopyFile(manifest, bak, dest, dest); err != nil { - return err - } - - // copy manifest to |dest| under temporary name - tmp := filepath.Join(doltDir, nomsDir, "temp-manifest") - if err = filesys.CopyFile(manifest, tmp, src, dest); err != nil { - return err - } - - // atomically swap the manifests - return dest.MoveFile(tmp, manifest) - // exit immediately! -} - func InitMigrationDB(ctx context.Context, existing *doltdb.DoltDB, src, dest filesys.Filesys) (err error) { base, err := src.Abs(".") if err != nil { @@ -177,8 +135,7 @@ func InitMigrationDB(ctx context.Context, existing *doltdb.DoltDB, src, dest fil } // migrate init commit - creation := ref.NewInternalRef(doltdb.CreationBranch) - init, err := existing.ResolveCommitRef(ctx, creation) + init, err := getOrCreateInternalRef(ctx, existing) if err != nil { return err } @@ -193,6 +150,7 @@ func InitMigrationDB(ctx context.Context, existing *doltdb.DoltDB, src, dest fil } nv := doltdb.HackNomsValuesFromRootValues(rv) + creation := ref.NewInternalRef(doltdb.CreationBranch) ds, err := db.GetDataset(ctx, creation.String()) if err != nil { return err @@ -202,6 +160,112 @@ func InitMigrationDB(ctx context.Context, existing *doltdb.DoltDB, src, dest fil return nil } +func getOrCreateInternalRef(ctx context.Context, existing *doltdb.DoltDB) (*doltdb.Commit, error) { + creation := ref.NewInternalRef(doltdb.CreationBranch) + + init, err := existing.ResolveCommitRef(ctx, creation) + if err == nil && init != nil { + return init, nil + } + if err != nil && !errors.Is(err, doltdb.ErrBranchNotFound) { + return nil, err + } + + heads, err := existing.GetBranches(ctx) + if err != nil { + return nil, err + } + cm, err := existing.ResolveCommitRef(ctx, heads[0]) + if err != nil { + return nil, err + } + + for { + if cm.NumParents() < 1 { + break + } + + cm, err = cm.GetParent(ctx, 0) + if err != nil { + return nil, err + } + } + + h, err := cm.HashOf() + if err != nil { + return nil, err + } + + err = existing.SetHead(ctx, creation, h) + if err != nil { + return nil, err + } + + return cm, nil +} + +func SwapChunkStores(ctx context.Context, menv Environment) error { + src, dest := menv.Migration.FS, menv.Existing.FS + + absSrc, err := src.Abs(filepath.Join(doltDir, nomsDir)) + if err != nil { + return err + } + + absDest, err := dest.Abs(filepath.Join(doltDir, nomsDir)) + if err != nil { + return err + } + + var cpErr error + err = src.Iter(absSrc, true, func(p string, size int64, isDir bool) (stop bool) { + if strings.Contains(p, manifestFile) || isDir { + return + } + + var relPath string + if relPath, cpErr = filepath.Rel(absSrc, p); cpErr != nil { + stop = true + return + } + + srcPath := filepath.Join(absSrc, relPath) + destPath := filepath.Join(absDest, relPath) + + if cpErr = filesys.CopyFile(srcPath, destPath, src, dest); cpErr != nil { + stop = true + } + return + }) + if err != nil { + return err + } + if cpErr != nil { + return cpErr + } + + return SwapManifests(ctx, src, dest) +} + +func SwapManifests(ctx context.Context, src, dest filesys.Filesys) (err error) { + // backup the current manifest + manifest := filepath.Join(doltDir, nomsDir, manifestFile) + bak := filepath.Join(doltDir, nomsDir, manifestFile+".bak") + if err = filesys.CopyFile(manifest, bak, dest, dest); err != nil { + return err + } + + // copy manifest to |dest| under temporary name + tmp := filepath.Join(doltDir, nomsDir, "temp-manifest") + if err = filesys.CopyFile(manifest, tmp, src, dest); err != nil { + return err + } + + // atomically swap the manifests + return dest.MoveFile(tmp, manifest) + // exit immediately! +} + func getMigrateFS(existing filesys.Filesys) (filesys.Filesys, error) { uniq := fmt.Sprintf("dolt_migration_%d", time.Now().Unix()) tmpPath := filepath.Join(existing.TempDir(), uniq) diff --git a/go/libraries/doltcore/migrate/progress.go b/go/libraries/doltcore/migrate/progress.go index 73a176b00d..e01603b1b1 100644 --- a/go/libraries/doltcore/migrate/progress.go +++ b/go/libraries/doltcore/migrate/progress.go @@ -16,6 +16,7 @@ package migrate import ( "context" + "fmt" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/store/hash" @@ -35,6 +36,8 @@ type CommitStack interface { type Progress interface { ChunkMapping CommitStack + + Log(ctx context.Context, format string, args ...any) } type memoryProgress struct { @@ -43,33 +46,33 @@ type memoryProgress struct { } func newProgress() Progress { - return memoryProgress{ + return &memoryProgress{ stack: make([]*doltdb.Commit, 0, 128), mapping: make(map[hash.Hash]hash.Hash, 128), } } -func (mem memoryProgress) Has(ctx context.Context, addr hash.Hash) (ok bool, err error) { +func (mem *memoryProgress) Has(ctx context.Context, addr hash.Hash) (ok bool, err error) { _, ok = mem.mapping[addr] return } -func (mem memoryProgress) Get(ctx context.Context, old hash.Hash) (new hash.Hash, err error) { +func (mem *memoryProgress) Get(ctx context.Context, old hash.Hash) (new hash.Hash, err error) { new = mem.mapping[old] return } -func (mem memoryProgress) Put(ctx context.Context, old, new hash.Hash) (err error) { +func (mem *memoryProgress) Put(ctx context.Context, old, new hash.Hash) (err error) { mem.mapping[old] = new return } -func (mem memoryProgress) Push(ctx context.Context, cm *doltdb.Commit) (err error) { +func (mem *memoryProgress) Push(ctx context.Context, cm *doltdb.Commit) (err error) { mem.stack = append(mem.stack, cm) return } -func (mem memoryProgress) Pop(ctx context.Context) (cm *doltdb.Commit, err error) { +func (mem *memoryProgress) Pop(ctx context.Context) (cm *doltdb.Commit, err error) { if len(mem.stack) == 0 { return nil, nil } @@ -78,3 +81,7 @@ func (mem memoryProgress) Pop(ctx context.Context) (cm *doltdb.Commit, err error mem.stack = mem.stack[:top] return } + +func (mem *memoryProgress) Log(ctx context.Context, format string, args ...any) { + fmt.Println(fmt.Sprintf(format, args...)) +} diff --git a/go/libraries/doltcore/migrate/transform.go b/go/libraries/doltcore/migrate/transform.go index 254c9d3119..9652405d94 100644 --- a/go/libraries/doltcore/migrate/transform.go +++ b/go/libraries/doltcore/migrate/transform.go @@ -27,6 +27,12 @@ import ( ) func MigrateCommit(ctx context.Context, r ref.DoltRef, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { + oldHash, err := cm.HashOf() + if err != nil { + return err + } + prog.Log(ctx, "migrating commit %s", oldHash.String()) + root, err := cm.GetRootValue(ctx) if err != nil { return err @@ -56,10 +62,6 @@ func MigrateCommit(ctx context.Context, r ref.DoltRef, cm *doltdb.Commit, new *d } // update progress - oldHash, err := cm.HashOf() - if err != nil { - return err - } newHash, err := migratedCm.HashOf() if err != nil { return err diff --git a/go/libraries/doltcore/migrate/traverse.go b/go/libraries/doltcore/migrate/traverse.go index 6c6a403df3..bce32cb2f6 100644 --- a/go/libraries/doltcore/migrate/traverse.go +++ b/go/libraries/doltcore/migrate/traverse.go @@ -25,24 +25,33 @@ import ( ) func TraverseDAG(ctx context.Context, old, new *doltdb.DoltDB) error { + prog := newProgress() + if err := MapInitCommits(ctx, old, new, prog); err != nil { + return err + } + heads, err := old.GetHeadRefs(ctx) if err != nil { return err } - prog := newProgress() for i := range heads { if err = TraverseRefHistory(ctx, heads[i], old, new, prog); err != nil { return err } } + if err = ValidateMigration(ctx, old, new); err != nil { + return err + } + return nil } -func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error { +func MapInitCommits(ctx context.Context, old, new *doltdb.DoltDB, prog Progress) error { // map init commits - o, err := old.ResolveCommitRef(ctx, ref.NewInternalRef(doltdb.CreationBranch)) + creation := ref.NewInternalRef(doltdb.CreationBranch) + o, err := old.ResolveCommitRef(ctx, creation) if err != nil { return err } @@ -50,7 +59,8 @@ func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.Dol if err != nil { return err } - n, err := new.ResolveCommitRef(ctx, ref.NewInternalRef(doltdb.CreationBranch)) + + n, err := new.ResolveCommitRef(ctx, creation) if err != nil { return err } @@ -58,13 +68,22 @@ func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.Dol if err != nil { return err } - if err = prog.Put(ctx, oh, nh); err != nil { - return err - } + + return prog.Put(ctx, oh, nh) +} + +func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error { switch r.GetType() { case ref.BranchRefType: - return TraverseBranchHistory(ctx, r.(ref.BranchRef), old, new, prog) + if err := TraverseBranchHistory(ctx, r, old, new, prog); err != nil { + return err + } + wsRef, err := ref.WorkingSetRefForHead(r) + if err != nil { + return err + } + return TraverseWorkingSetHistory(ctx, wsRef, old, new, prog) case ref.TagRefType: t, err := old.ResolveTag(ctx, r.(ref.TagRef)) @@ -74,7 +93,7 @@ func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.Dol return TraverseTagHistory(ctx, t, old, new, prog) case ref.RemoteRefType: - return nil // todo(andy) + return TraverseBranchHistory(ctx, r, old, new, prog) case ref.WorkspaceRefType: return nil // todo(andy) @@ -87,7 +106,7 @@ func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.Dol } } -func TraverseBranchHistory(ctx context.Context, r ref.BranchRef, old, new *doltdb.DoltDB, prog Progress) error { +func TraverseBranchHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error { cm, err := old.ResolveCommitRef(ctx, r) if err != nil { return err @@ -104,27 +123,15 @@ func TraverseBranchHistory(ctx context.Context, r ref.BranchRef, old, new *doltd if err != nil { return err } - if err = new.SetHead(ctx, r, newHash); err != nil { - return err - } - wsRef, err := ref.WorkingSetRefForHead(r) - if err != nil { - return err - } - - ws, err := old.ResolveWorkingSet(ctx, wsRef) - if err != nil { - return err - } - return TraverseWorkingSetHistory(ctx, ws, old, new, prog) + return new.SetHead(ctx, r, newHash) } func TraverseTagHistory(ctx context.Context, tag *doltdb.Tag, old, new *doltdb.DoltDB, prog Progress) error { return nil } -func TraverseWorkingSetHistory(ctx context.Context, ws *doltdb.WorkingSet, old, new *doltdb.DoltDB, prog Progress) error { +func TraverseWorkingSetHistory(ctx context.Context, wsRef ref.WorkingSetRef, old, new *doltdb.DoltDB, prog Progress) error { return nil } diff --git a/go/libraries/doltcore/migrate/validation.go b/go/libraries/doltcore/migrate/validation.go new file mode 100644 index 0000000000..5cd170297d --- /dev/null +++ b/go/libraries/doltcore/migrate/validation.go @@ -0,0 +1,48 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "context" + "fmt" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" +) + +func ValidateMigration(ctx context.Context, old, new *doltdb.DoltDB) error { + if err := validateBranchMapping(ctx, old, new); err != nil { + return err + } + return nil +} + +func validateBranchMapping(ctx context.Context, old, new *doltdb.DoltDB) error { + branches, err := old.GetBranches(ctx) + if err != nil { + return err + } + + var ok bool + for _, bref := range branches { + ok, err = new.HasBranch(ctx, bref.GetPath()) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("failed to map branch %s", bref.GetPath()) + } + } + return nil +} diff --git a/go/store/hash/hash.go b/go/store/hash/hash.go index 27e56d084a..8198caaf49 100644 --- a/go/store/hash/hash.go +++ b/go/store/hash/hash.go @@ -51,6 +51,7 @@ import ( "fmt" "regexp" "strconv" + "strings" "github.com/dolthub/dolt/go/store/d" ) @@ -191,3 +192,17 @@ func (hs HashSet) Empty() { delete(hs, h) } } + +func (hs HashSet) String() string { + var sb strings.Builder + sb.Grow(len(hs)*34 + 100) + + sb.WriteString("HashSet {\n") + for h := range hs { + sb.WriteString("\t") + sb.WriteString(h.String()) + sb.WriteString("\n") + } + sb.WriteString("}\n") + return sb.String() +} diff --git a/go/store/types/value_store.go b/go/store/types/value_store.go index 579880e5c6..9d4d2faaf9 100644 --- a/go/store/types/value_store.go +++ b/go/store/types/value_store.go @@ -95,7 +95,8 @@ func PanicIfDangling(ctx context.Context, unresolved hash.HashSet, cs chunks.Chu d.PanicIfError(err) if len(absent) != 0 { - d.Panic("Found dangling references to %v", absent) + s := absent.String() + d.Panic("Found dangling references to %s", s) } } From 3e571fb67a009a521608ac3772bdb8a87fda5d90 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 25 Jul 2022 13:15:04 -0700 Subject: [PATCH 08/15] copy blob tree along with map data --- go/libraries/doltcore/migrate/transform.go | 39 ++++++++++++++----- .../sqle/json/noms_json_value_test.go | 6 ++- go/store/cmd/noms/noms_show_test.go | 3 +- go/store/nbs/frag/main.go | 3 +- go/store/types/ref.go | 5 +-- 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/go/libraries/doltcore/migrate/transform.go b/go/libraries/doltcore/migrate/transform.go index 9652405d94..b83eeb5e34 100644 --- a/go/libraries/doltcore/migrate/transform.go +++ b/go/libraries/doltcore/migrate/transform.go @@ -186,13 +186,34 @@ func MigrateIndexSet(ctx context.Context, sch schema.Schema, oldSet durable.Inde func MigrateNomsMap(ctx context.Context, idx durable.Index, old, new types.ValueReadWriter) error { m := durable.NomsMapFromIndex(idx) - _, _, err := types.VisitMapLevelOrder(m, func(h hash.Hash) (int64, error) { - v, err := old.ReadValue(ctx, h) - if err != nil { - return 0, err - } - _, err = new.WriteValue(ctx, v) - return 0, err - }) - return err + return copyTreeFromValue(ctx, m, old, new) +} + +// copyTreeFromValue recursively copies |v| and all its children from |old| to |new|. +func copyTreeFromValue(ctx context.Context, v types.Value, old, new types.ValueReadWriter) error { + if _, err := new.WriteValue(ctx, v); err != nil { + return err + } + return types.WalkAddrs(v, old.Format(), func(h hash.Hash, isleaf bool) error { + if err := copyValue(ctx, h, old, new); err != nil { + return err + } + if isleaf { + return nil + } + val, err := old.ReadValue(ctx, h) + if err != nil { + return err + } + return copyTreeFromValue(ctx, val, old, new) + }) +} + +func copyValue(ctx context.Context, addr hash.Hash, old, new types.ValueReadWriter) (err error) { + var v types.Value + if v, err = old.ReadValue(ctx, addr); err != nil { + return err + } + _, err = new.WriteValue(ctx, v) + return } diff --git a/go/libraries/doltcore/sqle/json/noms_json_value_test.go b/go/libraries/doltcore/sqle/json/noms_json_value_test.go index fe8e624dac..8a7f63db7e 100644 --- a/go/libraries/doltcore/sqle/json/noms_json_value_test.go +++ b/go/libraries/doltcore/sqle/json/noms_json_value_test.go @@ -184,16 +184,18 @@ func TestJSONStructuralSharing(t *testing.T) { val := MustNomsJSONWithVRW(vrw, sb.String()) json_refs := make(hash.HashSet) - err := types.WalkAddrs(types.JSON(val), vrw.Format(), func(h hash.Hash, _ bool) { + err := types.WalkAddrs(types.JSON(val), vrw.Format(), func(h hash.Hash, _ bool) error { json_refs.Insert(h) + return nil }) require.NoError(t, err) tup, err := types.NewTuple(types.Format_Default, types.Int(12), types.JSON(val)) require.NoError(t, err) tuple_refs := make(hash.HashSet) - types.WalkAddrs(tup, vrw.Format(), func(h hash.Hash, _ bool) { + types.WalkAddrs(tup, vrw.Format(), func(h hash.Hash, _ bool) error { tuple_refs.Insert(h) + return nil }) assert.Greater(t, len(json_refs), 0) diff --git a/go/store/cmd/noms/noms_show_test.go b/go/store/cmd/noms/noms_show_test.go index 89ab65ed66..ad3a114181 100644 --- a/go/store/cmd/noms/noms_show_test.go +++ b/go/store/cmd/noms/noms_show_test.go @@ -152,8 +152,9 @@ func (s *nomsShowTestSuite) TestNomsShowRaw() { s.NoError(err) numChildChunks := 0 - err = types.WalkAddrs(l, vrw.Format(), func(_ hash.Hash, _ bool) { + err = types.WalkAddrs(l, vrw.Format(), func(_ hash.Hash, _ bool) error { numChildChunks++ + return nil }) s.NoError(err) s.True(numChildChunks > 0) diff --git a/go/store/nbs/frag/main.go b/go/store/nbs/frag/main.go index 143edc6067..860d9e36ba 100644 --- a/go/store/nbs/frag/main.go +++ b/go/store/nbs/frag/main.go @@ -130,11 +130,12 @@ func main() { orderedChildren := hash.HashSlice{} nextLevel := hash.HashSlice{} for _, h := range current { - _ = types.WalkAddrs(currentValues[h], types.Format_Default, func(h hash.Hash, isleaf bool) { + _ = types.WalkAddrs(currentValues[h], types.Format_Default, func(h hash.Hash, isleaf bool) error { orderedChildren = append(orderedChildren, h) if !visited[h] && !isleaf { nextLevel = append(nextLevel, h) } + return nil }) } diff --git a/go/store/types/ref.go b/go/store/types/ref.go index 175ee218b2..50e48b142b 100644 --- a/go/store/types/ref.go +++ b/go/store/types/ref.go @@ -242,9 +242,8 @@ func WalkAddrsForNBF(nbf *NomsBinFormat) func(chunks.Chunk, func(h hash.Hash, is } } -func WalkAddrs(v Value, nbf *NomsBinFormat, cb func(h hash.Hash, isleaf bool)) error { +func WalkAddrs(v Value, nbf *NomsBinFormat, cb func(h hash.Hash, isleaf bool) error) error { return v.walkRefs(nbf, func(r Ref) error { - cb(r.TargetHash(), r.Height() == 1) - return nil + return cb(r.TargetHash(), r.Height() == 1) }) } From a463bdb70a3b82ebe3576cb0a59246f77308d8eb Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 25 Jul 2022 14:51:49 -0700 Subject: [PATCH 09/15] refactored migration setup, deferring init-commit migration to traversal time --- go/libraries/doltcore/doltdb/doltdb.go | 4 ++ go/libraries/doltcore/doltdb/root_val.go | 8 +-- go/libraries/doltcore/migrate/environment.go | 65 ++++---------------- go/libraries/doltcore/migrate/transform.go | 62 ++++++++++++++++++- go/libraries/doltcore/migrate/traverse.go | 38 ++---------- 5 files changed, 83 insertions(+), 94 deletions(-) diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index d29a4282bf..d2779d7da2 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -79,6 +79,10 @@ func DoltDBFromCS(cs chunks.ChunkStore) *DoltDB { return &DoltDB{hooksDatabase{Database: db}, vrw, ns} } +func HackDatasDatabaseFromDoltDB(ddb *DoltDB) datas.Database { + return ddb.db +} + // LoadDoltDB will acquire a reference to the underlying noms db. If the Location is InMemDoltDB then a reference // to a newly created in memory database will be used. If the location is LocalDirDoltDB, the directory must exist or // this returns nil. diff --git a/go/libraries/doltcore/doltdb/root_val.go b/go/libraries/doltcore/doltdb/root_val.go index 746e1e96bd..1aa6729264 100644 --- a/go/libraries/doltcore/doltdb/root_val.go +++ b/go/libraries/doltcore/doltdb/root_val.go @@ -768,10 +768,6 @@ func (root *RootValue) withStorage(st rvStorage) *RootValue { return &RootValue{root.vrw, root.ns, st, nil} } -func HackNomsValuesFromRootValues(root *RootValue) types.Value { - return root.nomsValue() -} - func (root *RootValue) nomsValue() types.Value { return root.st.nomsValue() } @@ -1328,3 +1324,7 @@ func NewDataCacheKey(rv *RootValue) (DataCacheKey, error) { return DataCacheKey{hash}, nil } + +func HackNomsValuesFromRootValues(root *RootValue) types.Value { + return root.nomsValue() +} diff --git a/go/libraries/doltcore/migrate/environment.go b/go/libraries/doltcore/migrate/environment.go index 264c0a17c0..14076f9ca0 100644 --- a/go/libraries/doltcore/migrate/environment.go +++ b/go/libraries/doltcore/migrate/environment.go @@ -16,7 +16,6 @@ package migrate import ( "context" - "errors" "fmt" "path/filepath" "strings" @@ -36,9 +35,13 @@ const ( doltDir = dbfactory.DoltDir nomsDir = dbfactory.DataDir manifestFile = "manifest" + migrationRef = "migration" ) -var targetFormat = types.Format_DOLT_DEV +var ( + targetFormat = types.Format_DOLT_DEV + migrationMsg = fmt.Sprintf("migrating database to Noms Binary Format %s", targetFormat.VersionString()) +) type Environment struct { Migration *env.DoltEnv @@ -51,7 +54,7 @@ func NewEnvironment(ctx context.Context, existing *env.DoltEnv) (Environment, er return Environment{}, err } - if err = InitMigrationDB(ctx, existing.DoltDB, existing.FS, mfs); err != nil { + if err = InitMigrationDB(ctx, existing, existing.FS, mfs); err != nil { return Environment{}, err } @@ -81,7 +84,7 @@ func NewEnvironment(ctx context.Context, existing *env.DoltEnv) (Environment, er }, nil } -func InitMigrationDB(ctx context.Context, existing *doltdb.DoltDB, src, dest filesys.Filesys) (err error) { +func InitMigrationDB(ctx context.Context, existing *env.DoltEnv, src, dest filesys.Filesys) (err error) { base, err := src.Abs(".") if err != nil { return err @@ -134,24 +137,24 @@ func InitMigrationDB(ctx context.Context, existing *doltdb.DoltDB, src, dest fil return err } - // migrate init commit - init, err := getOrCreateInternalRef(ctx, existing) + // write init commit for migration + name, email, err := env.GetNameAndEmail(existing.Config) if err != nil { return err } - meta, err := init.GetCommitMeta(ctx) + meta, err := datas.NewCommitMeta(name, email, migrationMsg) if err != nil { return err } + rv, err := doltdb.EmptyRootValue(ctx, vrw, ns) if err != nil { return err } nv := doltdb.HackNomsValuesFromRootValues(rv) - creation := ref.NewInternalRef(doltdb.CreationBranch) - ds, err := db.GetDataset(ctx, creation.String()) + ds, err := db.GetDataset(ctx, ref.NewInternalRef(migrationRef).String()) if err != nil { return err } @@ -160,50 +163,6 @@ func InitMigrationDB(ctx context.Context, existing *doltdb.DoltDB, src, dest fil return nil } -func getOrCreateInternalRef(ctx context.Context, existing *doltdb.DoltDB) (*doltdb.Commit, error) { - creation := ref.NewInternalRef(doltdb.CreationBranch) - - init, err := existing.ResolveCommitRef(ctx, creation) - if err == nil && init != nil { - return init, nil - } - if err != nil && !errors.Is(err, doltdb.ErrBranchNotFound) { - return nil, err - } - - heads, err := existing.GetBranches(ctx) - if err != nil { - return nil, err - } - cm, err := existing.ResolveCommitRef(ctx, heads[0]) - if err != nil { - return nil, err - } - - for { - if cm.NumParents() < 1 { - break - } - - cm, err = cm.GetParent(ctx, 0) - if err != nil { - return nil, err - } - } - - h, err := cm.HashOf() - if err != nil { - return nil, err - } - - err = existing.SetHead(ctx, creation, h) - if err != nil { - return nil, err - } - - return cm, nil -} - func SwapChunkStores(ctx context.Context, menv Environment) error { src, dest := menv.Migration.FS, menv.Existing.FS diff --git a/go/libraries/doltcore/migrate/transform.go b/go/libraries/doltcore/migrate/transform.go index b83eeb5e34..d6c8bce710 100644 --- a/go/libraries/doltcore/migrate/transform.go +++ b/go/libraries/doltcore/migrate/transform.go @@ -17,20 +17,33 @@ package migrate import ( "context" + "github.com/dolthub/dolt/go/libraries/doltcore/ref" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" - "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/types" ) -func MigrateCommit(ctx context.Context, r ref.DoltRef, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { +func MigrateCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { oldHash, err := cm.HashOf() if err != nil { return err } + + ok, err := prog.Has(ctx, oldHash) + if err != nil { + return err + } else if ok { + return nil + } + + if cm.NumParents() == 0 { + return MigrateInitCommit(ctx, cm, new, prog) + } + prog.Log(ctx, "migrating commit %s", oldHash.String()) root, err := cm.GetRootValue(ctx) @@ -56,7 +69,7 @@ func MigrateCommit(ctx context.Context, r ref.DoltRef, cm *doltdb.Commit, new *d return err } - migratedCm, err := new.CommitValue(ctx, r, value, opts) + migratedCm, err := new.CommitDangling(ctx, value, opts) if err != nil { return err } @@ -70,11 +83,54 @@ func MigrateCommit(ctx context.Context, r ref.DoltRef, cm *doltdb.Commit, new *d return prog.Put(ctx, oldHash, newHash) } +func MigrateInitCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { + oldHash, err := cm.HashOf() + if err != nil { + return err + } + + rv, err := doltdb.EmptyRootValue(ctx, new.ValueReadWriter(), new.NodeStore()) + if err != nil { + return err + } + nv := doltdb.HackNomsValuesFromRootValues(rv) + + meta, err := cm.GetCommitMeta(ctx) + if err != nil { + return err + } + datasDB := doltdb.HackDatasDatabaseFromDoltDB(new) + + creation := ref.NewInternalRef(doltdb.CreationBranch) + ds, err := datasDB.GetDataset(ctx, creation.String()) + if err != nil { + return err + } + ds, err = datasDB.Commit(ctx, ds, nv, datas.CommitOptions{Meta: meta}) + if err != nil { + return err + } + + newCm, err := new.ResolveCommitRef(ctx, creation) + if err != nil { + return err + } + newHash, err := newCm.HashOf() + if err != nil { + return err + } + + return prog.Put(ctx, oldHash, newHash) +} + func MigrateCommitOptions(ctx context.Context, oldCm *doltdb.Commit, prog Progress) (datas.CommitOptions, error) { parents, err := oldCm.ParentHashes(ctx) if err != nil { return datas.CommitOptions{}, err } + if len(parents) == 0 { + panic("expected non-zero parents list") + } for i := range parents { migrated, err := prog.Get(ctx, parents[i]) diff --git a/go/libraries/doltcore/migrate/traverse.go b/go/libraries/doltcore/migrate/traverse.go index bce32cb2f6..8f47594726 100644 --- a/go/libraries/doltcore/migrate/traverse.go +++ b/go/libraries/doltcore/migrate/traverse.go @@ -25,16 +25,12 @@ import ( ) func TraverseDAG(ctx context.Context, old, new *doltdb.DoltDB) error { - prog := newProgress() - if err := MapInitCommits(ctx, old, new, prog); err != nil { - return err - } - heads, err := old.GetHeadRefs(ctx) if err != nil { return err } + prog := newProgress() for i := range heads { if err = TraverseRefHistory(ctx, heads[i], old, new, prog); err != nil { return err @@ -44,36 +40,10 @@ func TraverseDAG(ctx context.Context, old, new *doltdb.DoltDB) error { if err = ValidateMigration(ctx, old, new); err != nil { return err } - return nil } -func MapInitCommits(ctx context.Context, old, new *doltdb.DoltDB, prog Progress) error { - // map init commits - creation := ref.NewInternalRef(doltdb.CreationBranch) - o, err := old.ResolveCommitRef(ctx, creation) - if err != nil { - return err - } - oh, err := o.HashOf() - if err != nil { - return err - } - - n, err := new.ResolveCommitRef(ctx, creation) - if err != nil { - return err - } - nh, err := n.HashOf() - if err != nil { - return err - } - - return prog.Put(ctx, oh, nh) -} - func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error { - switch r.GetType() { case ref.BranchRefType: if err := TraverseBranchHistory(ctx, r, old, new, prog); err != nil { @@ -111,7 +81,7 @@ func TraverseBranchHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb. if err != nil { return err } - if err = TraverseCommitHistory(ctx, r, cm, new, prog); err != nil { + if err = TraverseCommitHistory(ctx, cm, new, prog); err != nil { return err } @@ -135,7 +105,7 @@ func TraverseWorkingSetHistory(ctx context.Context, wsRef ref.WorkingSetRef, old return nil } -func TraverseCommitHistory(ctx context.Context, r ref.DoltRef, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { +func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { ch, err := cm.HashOf() if err != nil { return err @@ -157,7 +127,7 @@ func TraverseCommitHistory(ctx context.Context, r ref.DoltRef, cm *doltdb.Commit } if idx < 0 { // parents for |cm| are done, migrate |cm| - if err = MigrateCommit(ctx, r, cm, new, prog); err != nil { + if err = MigrateCommit(ctx, cm, new, prog); err != nil { return err } // pop the stack, traverse upwards From a8d3024d7f14048d81c746039d66f90f84f3c5fa Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 25 Jul 2022 15:14:00 -0700 Subject: [PATCH 10/15] add support for tags and working sets --- go/libraries/doltcore/migrate/transform.go | 21 +++++++++++++++ go/libraries/doltcore/migrate/traverse.go | 30 ++++++++++------------ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/go/libraries/doltcore/migrate/transform.go b/go/libraries/doltcore/migrate/transform.go index d6c8bce710..89f1fe5a08 100644 --- a/go/libraries/doltcore/migrate/transform.go +++ b/go/libraries/doltcore/migrate/transform.go @@ -27,6 +27,27 @@ import ( "github.com/dolthub/dolt/go/store/types" ) +func MigrateWorkingSet(ctx context.Context, wsRef ref.WorkingSetRef, old, new *doltdb.DoltDB, prog Progress) error { + oldWs, err := old.ResolveWorkingSet(ctx, wsRef) + if err != nil { + return err + } + + wr, err := MigrateRoot(ctx, oldWs.WorkingRoot(), new) + if err != nil { + return err + } + + sr, err := MigrateRoot(ctx, oldWs.StagedRoot(), new) + if err != nil { + return err + } + + newWs := doltdb.EmptyWorkingSet(wsRef).WithWorkingRoot(wr).WithStagedRoot(sr) + + return new.UpdateWorkingSet(ctx, wsRef, newWs, hash.Hash{}, oldWs.Meta()) +} + func MigrateCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { oldHash, err := cm.HashOf() if err != nil { diff --git a/go/libraries/doltcore/migrate/traverse.go b/go/libraries/doltcore/migrate/traverse.go index 8f47594726..23ace3eb43 100644 --- a/go/libraries/doltcore/migrate/traverse.go +++ b/go/libraries/doltcore/migrate/traverse.go @@ -53,23 +53,16 @@ func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.Dol if err != nil { return err } - return TraverseWorkingSetHistory(ctx, wsRef, old, new, prog) + return MigrateWorkingSet(ctx, wsRef, old, new, prog) case ref.TagRefType: - t, err := old.ResolveTag(ctx, r.(ref.TagRef)) - if err != nil { - return err - } - return TraverseTagHistory(ctx, t, old, new, prog) + return TraverseTagHistory(ctx, r.(ref.TagRef), old, new, prog) case ref.RemoteRefType: return TraverseBranchHistory(ctx, r, old, new, prog) - case ref.WorkspaceRefType: - return nil // todo(andy) - - case ref.InternalRefType: - return nil // todo(andy)? + case ref.WorkspaceRefType, ref.InternalRefType: + return nil default: panic(fmt.Sprintf("unknown ref type %s", r.String())) @@ -97,12 +90,17 @@ func TraverseBranchHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb. return new.SetHead(ctx, r, newHash) } -func TraverseTagHistory(ctx context.Context, tag *doltdb.Tag, old, new *doltdb.DoltDB, prog Progress) error { - return nil -} +func TraverseTagHistory(ctx context.Context, r ref.TagRef, old, new *doltdb.DoltDB, prog Progress) error { + t, err := old.ResolveTag(ctx, r) + if err != nil { + return err + } -func TraverseWorkingSetHistory(ctx context.Context, wsRef ref.WorkingSetRef, old, new *doltdb.DoltDB, prog Progress) error { - return nil + if err = TraverseCommitHistory(ctx, t.Commit, new, prog); err != nil { + return err + } + + return new.NewTagAtCommit(ctx, r, t.Commit, t.Meta) } func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { From d018e126f288cc143c82273c823a034f5876f81d Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 25 Jul 2022 16:47:33 -0700 Subject: [PATCH 11/15] added migration bats --- integration-tests/bats/migrate.bats | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 integration-tests/bats/migrate.bats diff --git a/integration-tests/bats/migrate.bats b/integration-tests/bats/migrate.bats new file mode 100644 index 0000000000..afd8572a29 --- /dev/null +++ b/integration-tests/bats/migrate.bats @@ -0,0 +1,32 @@ +#!/usr/bin/env bats +load $BATS_TEST_DIRNAME/helper/common.bash + +setup() { + setup_common +} + +teardown() { + teardown_common +} + +function checksum_table { + QUERY="SELECT GROUP_CONCAT(column_name) FROM information_schema.columns WHERE table_name = '$1'" + COLUMNS=$( dolt sql -q "$QUERY" -r csv | tail -n1 | sed 's/"//g' ) + dolt sql -q "SELECT CAST(SUM(CRC32(CONCAT($COLUMNS))) AS UNSIGNED) FROM $1" -r csv | tail -n1 +} + +@test "migrate: smoke test" { + dolt sql < Date: Mon, 25 Jul 2022 20:05:04 -0700 Subject: [PATCH 12/15] added more bats, fixed tag migration --- go/libraries/doltcore/doltdb/doltdb.go | 9 ++ go/libraries/doltcore/migrate/environment.go | 2 +- go/libraries/doltcore/migrate/traverse.go | 14 ++- integration-tests/bats/migrate.bats | 112 +++++++++++++++++-- 4 files changed, 128 insertions(+), 9 deletions(-) diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index d2779d7da2..ad0e7aca36 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -417,6 +417,15 @@ func (ddb *DoltDB) ReadRootValue(ctx context.Context, h hash.Hash) (*RootValue, return decodeRootNomsValue(ddb.vrw, ddb.ns, val) } +// ReadCommit reads the Commit whose hash is |h|, if one exists. +func (ddb *DoltDB) ReadCommit(ctx context.Context, h hash.Hash) (*Commit, error) { + c, err := datas.LoadCommitAddr(ctx, ddb.vrw, h) + if err != nil { + return nil, err + } + return NewCommit(ctx, ddb.vrw, ddb.ns, c) +} + // Commit will update a branch's head value to be that of a previously committed root value hash func (ddb *DoltDB) Commit(ctx context.Context, valHash hash.Hash, dref ref.DoltRef, cm *datas.CommitMeta) (*Commit, error) { if dref.GetType() != ref.BranchRefType { diff --git a/go/libraries/doltcore/migrate/environment.go b/go/libraries/doltcore/migrate/environment.go index 14076f9ca0..1249087d29 100644 --- a/go/libraries/doltcore/migrate/environment.go +++ b/go/libraries/doltcore/migrate/environment.go @@ -226,7 +226,7 @@ func SwapManifests(ctx context.Context, src, dest filesys.Filesys) (err error) { } func getMigrateFS(existing filesys.Filesys) (filesys.Filesys, error) { - uniq := fmt.Sprintf("dolt_migration_%d", time.Now().Unix()) + uniq := fmt.Sprintf("dolt_migration_%d", time.Now().UnixNano()) tmpPath := filepath.Join(existing.TempDir(), uniq) if err := existing.MkDirs(tmpPath); err != nil { return nil, err diff --git a/go/libraries/doltcore/migrate/traverse.go b/go/libraries/doltcore/migrate/traverse.go index 23ace3eb43..ff428aaf6b 100644 --- a/go/libraries/doltcore/migrate/traverse.go +++ b/go/libraries/doltcore/migrate/traverse.go @@ -100,7 +100,19 @@ func TraverseTagHistory(ctx context.Context, r ref.TagRef, old, new *doltdb.Dolt return err } - return new.NewTagAtCommit(ctx, r, t.Commit, t.Meta) + oldHash, err := t.Commit.HashOf() + if err != nil { + return err + } + newHash, err := prog.Get(ctx, oldHash) + if err != nil { + return err + } + cm, err := new.ReadCommit(ctx, newHash) + if err != nil { + return err + } + return new.NewTagAtCommit(ctx, r, cm, t.Meta) } func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { diff --git a/integration-tests/bats/migrate.bats b/integration-tests/bats/migrate.bats index afd8572a29..5c08199a7d 100644 --- a/integration-tests/bats/migrate.bats +++ b/integration-tests/bats/migrate.bats @@ -2,6 +2,7 @@ load $BATS_TEST_DIRNAME/helper/common.bash setup() { + TARGET_NBF="__DOLT_DEV__" setup_common } @@ -12,21 +13,118 @@ teardown() { function checksum_table { QUERY="SELECT GROUP_CONCAT(column_name) FROM information_schema.columns WHERE table_name = '$1'" COLUMNS=$( dolt sql -q "$QUERY" -r csv | tail -n1 | sed 's/"//g' ) - dolt sql -q "SELECT CAST(SUM(CRC32(CONCAT($COLUMNS))) AS UNSIGNED) FROM $1" -r csv | tail -n1 + dolt sql -q "SELECT CAST(SUM(CRC32(CONCAT($COLUMNS))) AS UNSIGNED) FROM $1 AS OF '$2';" -r csv | tail -n1 } @test "migrate: smoke test" { dolt sql < Date: Mon, 25 Jul 2022 20:06:27 -0700 Subject: [PATCH 13/15] skip migrationg bats for __DOLT_1__ --- integration-tests/bats/migrate.bats | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration-tests/bats/migrate.bats b/integration-tests/bats/migrate.bats index 5c08199a7d..b120a9fd7f 100644 --- a/integration-tests/bats/migrate.bats +++ b/integration-tests/bats/migrate.bats @@ -2,6 +2,8 @@ load $BATS_TEST_DIRNAME/helper/common.bash setup() { + skip_nbf_dolt_1 + TARGET_NBF="__DOLT_DEV__" setup_common } From 4c3ab5f21f5120f247b7553ad6dd26f0a924d693 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Tue, 26 Jul 2022 07:53:56 -0700 Subject: [PATCH 14/15] also skip migrate bats for DOLT_DEV --- go/libraries/doltcore/doltdb/doltdb.go | 2 ++ go/libraries/doltcore/doltdb/root_val.go | 2 ++ integration-tests/bats/helper/common.bash | 6 ++++++ integration-tests/bats/migrate.bats | 1 + 4 files changed, 11 insertions(+) diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index ad0e7aca36..f9f4ce7a90 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -79,6 +79,8 @@ func DoltDBFromCS(cs chunks.ChunkStore) *DoltDB { return &DoltDB{hooksDatabase{Database: db}, vrw, ns} } +// HackDatasDatabaseFromDoltDB unwraps a DoltDB to a datas.Database. +// Deprecated: only for use in dolt migrate. func HackDatasDatabaseFromDoltDB(ddb *DoltDB) datas.Database { return ddb.db } diff --git a/go/libraries/doltcore/doltdb/root_val.go b/go/libraries/doltcore/doltdb/root_val.go index 1aa6729264..f05bcc526f 100644 --- a/go/libraries/doltcore/doltdb/root_val.go +++ b/go/libraries/doltcore/doltdb/root_val.go @@ -1325,6 +1325,8 @@ func NewDataCacheKey(rv *RootValue) (DataCacheKey, error) { return DataCacheKey{hash}, nil } +// HackNomsValuesFromRootValues unwraps a RootVal to a noms Value. +// Deprecated: only for use in dolt migrate. func HackNomsValuesFromRootValues(root *RootValue) types.Value { return root.nomsValue() } diff --git a/integration-tests/bats/helper/common.bash b/integration-tests/bats/helper/common.bash index 6693654b15..e2259ae925 100644 --- a/integration-tests/bats/helper/common.bash +++ b/integration-tests/bats/helper/common.bash @@ -69,6 +69,12 @@ skip_nbf_dolt_1() { fi } +skip_nbf_dolt_dev() { + if [ "$DOLT_DEFAULT_BIN_FORMAT" = "__DOLT_DEV__" ]; then + skip "skipping test for nomsBinFormat __DOLT_DEV__" + fi +} + setup_common() { setup_no_dolt_init dolt init diff --git a/integration-tests/bats/migrate.bats b/integration-tests/bats/migrate.bats index b120a9fd7f..1505ae7b98 100644 --- a/integration-tests/bats/migrate.bats +++ b/integration-tests/bats/migrate.bats @@ -3,6 +3,7 @@ load $BATS_TEST_DIRNAME/helper/common.bash setup() { skip_nbf_dolt_1 + skip_nbf_dolt_dev TARGET_NBF="__DOLT_DEV__" setup_common From cdfdad3ca7f4f4ccb015efd1cb2922403c12929c Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Tue, 26 Jul 2022 11:56:04 -0700 Subject: [PATCH 15/15] pr feedback --- go/cmd/dolt/commands/migrate.go | 1 + go/libraries/doltcore/doltdb/doltdb.go | 1 + go/libraries/doltcore/migrate/environment.go | 11 +++--- go/libraries/doltcore/migrate/transform.go | 34 +++++++++---------- go/libraries/doltcore/migrate/traverse.go | 27 ++++++++------- go/libraries/doltcore/migrate/validation.go | 2 +- .../sqle/json/noms_json_value_test.go | 3 +- 7 files changed, 43 insertions(+), 36 deletions(-) diff --git a/go/cmd/dolt/commands/migrate.go b/go/cmd/dolt/commands/migrate.go index ad6edce6fe..36062692df 100644 --- a/go/cmd/dolt/commands/migrate.go +++ b/go/cmd/dolt/commands/migrate.go @@ -93,6 +93,7 @@ func (cmd MigrateCmd) Exec(ctx context.Context, commandStr string, args []string return 0 } +// MigrateDatabase migrates the NomsBinFormat of |dEnv.DoltDB|. func MigrateDatabase(ctx context.Context, dEnv *env.DoltEnv) error { menv, err := migrate.NewEnvironment(ctx, dEnv) if err != nil { diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index f9f4ce7a90..9f389acf18 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -596,6 +596,7 @@ func (ddb *DoltDB) CommitDanglingWithParentCommits(ctx context.Context, valHash return ddb.CommitDangling(ctx, val, commitOpts) } +// CommitDangling creates a new Commit for |val| that is not referenced by any DoltRef. func (ddb *DoltDB) CommitDangling(ctx context.Context, val types.Value, opts datas.CommitOptions) (*Commit, error) { cs := datas.ChunkStoreFromDatabase(ddb.db) diff --git a/go/libraries/doltcore/migrate/environment.go b/go/libraries/doltcore/migrate/environment.go index 1249087d29..7578b78f83 100644 --- a/go/libraries/doltcore/migrate/environment.go +++ b/go/libraries/doltcore/migrate/environment.go @@ -43,18 +43,20 @@ var ( migrationMsg = fmt.Sprintf("migrating database to Noms Binary Format %s", targetFormat.VersionString()) ) +// Environment is a migration environment. type Environment struct { Migration *env.DoltEnv Existing *env.DoltEnv } +// NewEnvironment creates a migration Environment for |existing|. func NewEnvironment(ctx context.Context, existing *env.DoltEnv) (Environment, error) { mfs, err := getMigrateFS(existing.FS) if err != nil { return Environment{}, err } - if err = InitMigrationDB(ctx, existing, existing.FS, mfs); err != nil { + if err = initMigrationDB(ctx, existing, existing.FS, mfs); err != nil { return Environment{}, err } @@ -84,7 +86,7 @@ func NewEnvironment(ctx context.Context, existing *env.DoltEnv) (Environment, er }, nil } -func InitMigrationDB(ctx context.Context, existing *env.DoltEnv, src, dest filesys.Filesys) (err error) { +func initMigrationDB(ctx context.Context, existing *env.DoltEnv, src, dest filesys.Filesys) (err error) { base, err := src.Abs(".") if err != nil { return err @@ -163,6 +165,7 @@ func InitMigrationDB(ctx context.Context, existing *env.DoltEnv, src, dest files return nil } +// SwapChunkStores atomically swaps the ChunkStores of |menv.Migration| and |menv.Existing|. func SwapChunkStores(ctx context.Context, menv Environment) error { src, dest := menv.Migration.FS, menv.Existing.FS @@ -203,10 +206,10 @@ func SwapChunkStores(ctx context.Context, menv Environment) error { return cpErr } - return SwapManifests(ctx, src, dest) + return swapManifests(ctx, src, dest) } -func SwapManifests(ctx context.Context, src, dest filesys.Filesys) (err error) { +func swapManifests(ctx context.Context, src, dest filesys.Filesys) (err error) { // backup the current manifest manifest := filepath.Join(doltDir, nomsDir, manifestFile) bak := filepath.Join(doltDir, nomsDir, manifestFile+".bak") diff --git a/go/libraries/doltcore/migrate/transform.go b/go/libraries/doltcore/migrate/transform.go index 89f1fe5a08..541beb0e09 100644 --- a/go/libraries/doltcore/migrate/transform.go +++ b/go/libraries/doltcore/migrate/transform.go @@ -27,18 +27,18 @@ import ( "github.com/dolthub/dolt/go/store/types" ) -func MigrateWorkingSet(ctx context.Context, wsRef ref.WorkingSetRef, old, new *doltdb.DoltDB, prog Progress) error { +func migrateWorkingSet(ctx context.Context, wsRef ref.WorkingSetRef, old, new *doltdb.DoltDB, prog Progress) error { oldWs, err := old.ResolveWorkingSet(ctx, wsRef) if err != nil { return err } - wr, err := MigrateRoot(ctx, oldWs.WorkingRoot(), new) + wr, err := migrateRoot(ctx, oldWs.WorkingRoot(), new) if err != nil { return err } - sr, err := MigrateRoot(ctx, oldWs.StagedRoot(), new) + sr, err := migrateRoot(ctx, oldWs.StagedRoot(), new) if err != nil { return err } @@ -48,7 +48,7 @@ func MigrateWorkingSet(ctx context.Context, wsRef ref.WorkingSetRef, old, new *d return new.UpdateWorkingSet(ctx, wsRef, newWs, hash.Hash{}, oldWs.Meta()) } -func MigrateCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { +func migrateCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { oldHash, err := cm.HashOf() if err != nil { return err @@ -62,7 +62,7 @@ func MigrateCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, p } if cm.NumParents() == 0 { - return MigrateInitCommit(ctx, cm, new, prog) + return migrateInitCommit(ctx, cm, new, prog) } prog.Log(ctx, "migrating commit %s", oldHash.String()) @@ -72,7 +72,7 @@ func MigrateCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, p return err } - mRoot, err := MigrateRoot(ctx, root, new) + mRoot, err := migrateRoot(ctx, root, new) if err != nil { return err } @@ -85,7 +85,7 @@ func MigrateCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, p return err } - opts, err := MigrateCommitOptions(ctx, cm, prog) + opts, err := migrateCommitOptions(ctx, cm, prog) if err != nil { return err } @@ -104,7 +104,7 @@ func MigrateCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, p return prog.Put(ctx, oldHash, newHash) } -func MigrateInitCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { +func migrateInitCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { oldHash, err := cm.HashOf() if err != nil { return err @@ -144,7 +144,7 @@ func MigrateInitCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltD return prog.Put(ctx, oldHash, newHash) } -func MigrateCommitOptions(ctx context.Context, oldCm *doltdb.Commit, prog Progress) (datas.CommitOptions, error) { +func migrateCommitOptions(ctx context.Context, oldCm *doltdb.Commit, prog Progress) (datas.CommitOptions, error) { parents, err := oldCm.ParentHashes(ctx) if err != nil { return datas.CommitOptions{}, err @@ -172,7 +172,7 @@ func MigrateCommitOptions(ctx context.Context, oldCm *doltdb.Commit, prog Progre }, nil } -func MigrateRoot(ctx context.Context, root *doltdb.RootValue, new *doltdb.DoltDB) (*doltdb.RootValue, error) { +func migrateRoot(ctx context.Context, root *doltdb.RootValue, new *doltdb.DoltDB) (*doltdb.RootValue, error) { migrated, err := doltdb.EmptyRootValue(ctx, new.ValueReadWriter(), new.NodeStore()) if err != nil { return nil, err @@ -189,7 +189,7 @@ func MigrateRoot(ctx context.Context, root *doltdb.RootValue, new *doltdb.DoltDB } err = root.IterTables(ctx, func(name string, tbl *doltdb.Table, _ schema.Schema) (bool, error) { - mtbl, err := MigrateTable(ctx, tbl, new) + mtbl, err := migrateTable(ctx, tbl, new) if err != nil { return true, err } @@ -207,13 +207,13 @@ func MigrateRoot(ctx context.Context, root *doltdb.RootValue, new *doltdb.DoltDB return migrated, nil } -func MigrateTable(ctx context.Context, table *doltdb.Table, new *doltdb.DoltDB) (*doltdb.Table, error) { +func migrateTable(ctx context.Context, table *doltdb.Table, new *doltdb.DoltDB) (*doltdb.Table, error) { rows, err := table.GetRowData(ctx) if err != nil { return nil, err } - err = MigrateNomsMap(ctx, rows, table.ValueReadWriter(), new.ValueReadWriter()) + err = migrateNomsMap(ctx, rows, table.ValueReadWriter(), new.ValueReadWriter()) if err != nil { return nil, err } @@ -234,7 +234,7 @@ func MigrateTable(ctx context.Context, table *doltdb.Table, new *doltdb.DoltDB) return nil, err } - newSet, err := MigrateIndexSet(ctx, sch, oldSet, table.ValueReadWriter(), new) + newSet, err := migrateIndexSet(ctx, sch, oldSet, table.ValueReadWriter(), new) if err != nil { return nil, err } @@ -242,14 +242,14 @@ func MigrateTable(ctx context.Context, table *doltdb.Table, new *doltdb.DoltDB) return doltdb.NewTable(ctx, new.ValueReadWriter(), new.NodeStore(), sch, rows, newSet, autoInc) } -func MigrateIndexSet(ctx context.Context, sch schema.Schema, oldSet durable.IndexSet, old types.ValueReadWriter, new *doltdb.DoltDB) (durable.IndexSet, error) { +func migrateIndexSet(ctx context.Context, sch schema.Schema, oldSet durable.IndexSet, old types.ValueReadWriter, new *doltdb.DoltDB) (durable.IndexSet, error) { newSet := durable.NewIndexSet(ctx, new.ValueReadWriter(), new.NodeStore()) for _, def := range sch.Indexes().AllIndexes() { idx, err := oldSet.GetIndex(ctx, sch, def.Name()) if err != nil { return nil, err } - if err = MigrateNomsMap(ctx, idx, old, new.ValueReadWriter()); err != nil { + if err = migrateNomsMap(ctx, idx, old, new.ValueReadWriter()); err != nil { return nil, err } @@ -261,7 +261,7 @@ func MigrateIndexSet(ctx context.Context, sch schema.Schema, oldSet durable.Inde return newSet, nil } -func MigrateNomsMap(ctx context.Context, idx durable.Index, old, new types.ValueReadWriter) error { +func migrateNomsMap(ctx context.Context, idx durable.Index, old, new types.ValueReadWriter) error { m := durable.NomsMapFromIndex(idx) return copyTreeFromValue(ctx, m, old, new) } diff --git a/go/libraries/doltcore/migrate/traverse.go b/go/libraries/doltcore/migrate/traverse.go index ff428aaf6b..b9313781c8 100644 --- a/go/libraries/doltcore/migrate/traverse.go +++ b/go/libraries/doltcore/migrate/traverse.go @@ -24,6 +24,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/ref" ) +// TraverseDAG traverses |old|, migrating values to |new|. func TraverseDAG(ctx context.Context, old, new *doltdb.DoltDB) error { heads, err := old.GetHeadRefs(ctx) if err != nil { @@ -32,34 +33,34 @@ func TraverseDAG(ctx context.Context, old, new *doltdb.DoltDB) error { prog := newProgress() for i := range heads { - if err = TraverseRefHistory(ctx, heads[i], old, new, prog); err != nil { + if err = traverseRefHistory(ctx, heads[i], old, new, prog); err != nil { return err } } - if err = ValidateMigration(ctx, old, new); err != nil { + if err = validateMigration(ctx, old, new); err != nil { return err } return nil } -func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error { +func traverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error { switch r.GetType() { case ref.BranchRefType: - if err := TraverseBranchHistory(ctx, r, old, new, prog); err != nil { + if err := traverseBranchHistory(ctx, r, old, new, prog); err != nil { return err } wsRef, err := ref.WorkingSetRefForHead(r) if err != nil { return err } - return MigrateWorkingSet(ctx, wsRef, old, new, prog) + return migrateWorkingSet(ctx, wsRef, old, new, prog) case ref.TagRefType: - return TraverseTagHistory(ctx, r.(ref.TagRef), old, new, prog) + return traverseTagHistory(ctx, r.(ref.TagRef), old, new, prog) case ref.RemoteRefType: - return TraverseBranchHistory(ctx, r, old, new, prog) + return traverseBranchHistory(ctx, r, old, new, prog) case ref.WorkspaceRefType, ref.InternalRefType: return nil @@ -69,12 +70,12 @@ func TraverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.Dol } } -func TraverseBranchHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error { +func traverseBranchHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error { cm, err := old.ResolveCommitRef(ctx, r) if err != nil { return err } - if err = TraverseCommitHistory(ctx, cm, new, prog); err != nil { + if err = traverseCommitHistory(ctx, cm, new, prog); err != nil { return err } @@ -90,13 +91,13 @@ func TraverseBranchHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb. return new.SetHead(ctx, r, newHash) } -func TraverseTagHistory(ctx context.Context, r ref.TagRef, old, new *doltdb.DoltDB, prog Progress) error { +func traverseTagHistory(ctx context.Context, r ref.TagRef, old, new *doltdb.DoltDB, prog Progress) error { t, err := old.ResolveTag(ctx, r) if err != nil { return err } - if err = TraverseCommitHistory(ctx, t.Commit, new, prog); err != nil { + if err = traverseCommitHistory(ctx, t.Commit, new, prog); err != nil { return err } @@ -115,7 +116,7 @@ func TraverseTagHistory(ctx context.Context, r ref.TagRef, old, new *doltdb.Dolt return new.NewTagAtCommit(ctx, r, cm, t.Meta) } -func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { +func traverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error { ch, err := cm.HashOf() if err != nil { return err @@ -137,7 +138,7 @@ func TraverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.D } if idx < 0 { // parents for |cm| are done, migrate |cm| - if err = MigrateCommit(ctx, cm, new, prog); err != nil { + if err = migrateCommit(ctx, cm, new, prog); err != nil { return err } // pop the stack, traverse upwards diff --git a/go/libraries/doltcore/migrate/validation.go b/go/libraries/doltcore/migrate/validation.go index 5cd170297d..c72599bb66 100644 --- a/go/libraries/doltcore/migrate/validation.go +++ b/go/libraries/doltcore/migrate/validation.go @@ -21,7 +21,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" ) -func ValidateMigration(ctx context.Context, old, new *doltdb.DoltDB) error { +func validateMigration(ctx context.Context, old, new *doltdb.DoltDB) error { if err := validateBranchMapping(ctx, old, new); err != nil { return err } diff --git a/go/libraries/doltcore/sqle/json/noms_json_value_test.go b/go/libraries/doltcore/sqle/json/noms_json_value_test.go index 8a7f63db7e..93d6aa1e85 100644 --- a/go/libraries/doltcore/sqle/json/noms_json_value_test.go +++ b/go/libraries/doltcore/sqle/json/noms_json_value_test.go @@ -193,10 +193,11 @@ func TestJSONStructuralSharing(t *testing.T) { tup, err := types.NewTuple(types.Format_Default, types.Int(12), types.JSON(val)) require.NoError(t, err) tuple_refs := make(hash.HashSet) - types.WalkAddrs(tup, vrw.Format(), func(h hash.Hash, _ bool) error { + err = types.WalkAddrs(tup, vrw.Format(), func(h hash.Hash, _ bool) error { tuple_refs.Insert(h) return nil }) + assert.NoError(t, err) assert.Greater(t, len(json_refs), 0) assert.Equal(t, len(json_refs), len(tuple_refs))